diff --git a/index-recovery-tests.md b/index-recovery-tests.md new file mode 100644 index 000000000000..3e6fbcab19ce --- /dev/null +++ b/index-recovery-tests.md @@ -0,0 +1,15 @@ +# Harder-to-Reproduce Index Recovery Performance Results + +I tested recovery of 1 and 12 shards of ~20 Gigs each. The size makes it a bit challenging to package nicely in a reproducible benchmark, although I am sure it can be done. +I am confident you can reproduce this behavior with a comparable amount of data and cloud structure. I can share the scripts I used to achieve these results if it is helpful. + +## Results Summary + +| Scenario | Shards | Configuration | Result | Time | +|----------|--------|---------------|--------|------| +| HTTP/2 | 1 | default | Fast | ~40s | +| HTTP/1 | 1 | default | Fast | ~50s | +| HTTP/1 | 12 | default | Fast | ~90s | +| HTTP/2 | 12 | default | Slowest | ~320s | +| HTTP/2 | 12 | `maxConcurrentStreams=1`| Slower | ~180s | + diff --git a/scripts/add-replicas.sh b/scripts/add-replicas.sh new file mode 100644 index 000000000000..f2ee859fd18d --- /dev/null +++ b/scripts/add-replicas.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# ============================================================================= +# Script to add replicas to a target node +# +# Usage: +# ./add-replicas.sh [SOLR_URL] [COLLECTION] [TARGET_NODE] [COUNT] [TYPE] +# +# Example: +# ./add-replicas.sh http://localhost:8983/solr test solr2:8983_solr 12 TLOG +# ./add-replicas.sh http://localhost:8983/solr test solr2:8983_solr 1 NRT +# ============================================================================= + +set -e + +SOLR_URL="${1:-http://localhost:8983/solr}" +COLLECTION="${2:-test}" +TARGET_NODE="${3:-solr2:8983_solr}" +NUM_SHARDS="${4:-12}" +TYPE="${5:-TLOG}" + +echo "Ensuring $NUM_SHARDS shards with 1 replica of type $TYPE on $TARGET_NODE for collection $COLLECTION" + +# Fetch cluster status +echo "Fetching cluster status from $SOLR_URL..." +cluster_status=$(curl -s "$SOLR_URL/admin/collections?action=CLUSTERSTATUS") + +# Validate JSON response +if ! echo "$cluster_status" | jq -e . >/dev/null 2>&1; then + echo "Error: Invalid JSON response from Solr." + echo "Response: $cluster_status" + exit 1 +fi + +# Check if collection exists +if echo "$cluster_status" | jq -e ".cluster.collections[\"$COLLECTION\"] == null" >/dev/null; then + echo "Collection '$COLLECTION' not found." + echo "Creating collection '$COLLECTION' with $NUM_SHARDS shards..." + + # Determine replica types for CREATE + # prioritizing TLOG if requested + CREATE_PARAMS="action=CREATE&name=$COLLECTION&numShards=$NUM_SHARDS" + + if [ "$TYPE" == "TLOG" ]; then + CREATE_PARAMS="${CREATE_PARAMS}&nrtReplicas=0&tlogReplicas=1" + elif [ "$TYPE" == "PULL" ]; then + CREATE_PARAMS="${CREATE_PARAMS}&nrtReplicas=0&pullReplicas=1" + else + CREATE_PARAMS="${CREATE_PARAMS}&replicationFactor=1" + fi + + # Create collection targeted at the node to ensure initial replicas are there + create_response=$(curl -s -w "\n%{http_code}" \ + "$SOLR_URL/admin/collections?${CREATE_PARAMS}&createNodeSet=$TARGET_NODE") + + create_http_code=$(echo "$create_response" | tail -n1) + + if [ "$create_http_code" != "200" ]; then + echo "Error creating collection: HTTP $create_http_code" + echo "$create_response" | head -n -1 + exit 1 + fi + + echo "Collection created successfully." + + # We are done since CREATE with createNodeSet puts them there + exit 0 +fi + +echo "Collection '$COLLECTION' exists. Checking shards..." + +# Refresh cluster status +cluster_status=$(curl -s "$SOLR_URL/admin/collections?action=CLUSTERSTATUS") + +# Iterate through expected shards 1..NUM_SHARDS +for ((i=1; i<=NUM_SHARDS; i++)); do + shard_name="shard${i}" + + # Check if shard exists + shard_exists=$(echo "$cluster_status" | jq -r ".cluster.collections[\"$COLLECTION\"].shards[\"$shard_name\"] // empty") + + if [ -z "$shard_exists" ]; then + echo " $shard_name does not exist. Creating..." + + # Create shard + response=$(curl -s -w "\n%{http_code}" \ + "$SOLR_URL/admin/collections?action=CREATESHARD&collection=$COLLECTION&shard=$shard_name&createNodeSet=$TARGET_NODE") + + # CREATESHARD doesn't take type params easily for the new replica, it usually uses collection defaults. + # But if we use createNodeSet it creates a replica there. + # However, checking if it created the right TYPE is hard atomically. + # Typically CREATESHARD adds replicas based on collection settings. + + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" != "200" ]; then + echo " Error creating shard: HTTP $http_code" + echo "$response" | head -n -1 + exit 1 + fi + echo " $shard_name created." + + # We might need to ensure the type is correct if default isn't TLOG. + # But for now assuming collection settings or manual add later if needed. + # Ideally we'd check and delete/re-add if wrong type, but that's complex. + else + # Shard exists, check for replica on TARGET_NODE + # We look for a replica on this node + replicas_on_node=$(echo "$cluster_status" | jq -r ".cluster.collections[\"$COLLECTION\"].shards[\"$shard_name\"].replicas | to_entries[] | select(.value.node_name == \"$TARGET_NODE\") | .key") + + if [ -z "$replicas_on_node" ]; then + echo " $shard_name exists but has no replica on $TARGET_NODE. Adding $TYPE replica..." + + response=$(curl -s -w "\n%{http_code}" \ + "$SOLR_URL/admin/collections?action=ADDREPLICA&collection=$COLLECTION&shard=$shard_name&node=$TARGET_NODE&type=$TYPE") + + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" != "200" ]; then + echo " Error adding replica: HTTP $http_code" + echo "$response" | head -n -1 + exit 1 + fi + echo " Replica added." + else + echo " $shard_name already has replica on $TARGET_NODE. Skipping." + fi + fi +done + +echo "" +echo "=========================================" +echo "Configuration complete!" +echo "Collection: $COLLECTION" +echo "Target node: $TARGET_NODE" +echo "Shards checked: $NUM_SHARDS" +echo "=========================================" + diff --git a/scripts/cycle-replicas.sh b/scripts/cycle-replicas.sh new file mode 100644 index 000000000000..f58ec12baafa --- /dev/null +++ b/scripts/cycle-replicas.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# ============================================================================= +# Script to remove all replicas from a node and then add them back +# +# Usage: +# ./cycle-replicas.sh [SOLR_URL] [COLLECTION] [TARGET_NODE] +# +# Example: +# ./cycle-replicas.sh http://localhost:8983/solr test solr2:8983_solr +# ============================================================================= + +set -e + +SOLR_URL="${1:-http://localhost:8983/solr}" +COLLECTION="${2:-test}" +TARGET_NODE="${3:-solr2:8983_solr}" + +echo "Cycling replicas on $TARGET_NODE for collection $COLLECTION" +echo "" + +# Get cluster status +cluster_status=$(curl -s "$SOLR_URL/admin/collections?action=CLUSTERSTATUS") + +# Find all replicas on the target node +# Format: shard_name:replica_name +replicas_on_node=$(echo "$cluster_status" | jq -r " + .cluster.collections[\"$COLLECTION\"].shards | to_entries[] | + .key as \$shard | + .value.replicas | to_entries[] | + select(.value.node_name == \"$TARGET_NODE\") | + \"\(\$shard):\(.key)\" +") + +if [ -z "$replicas_on_node" ]; then + echo "No replicas found on $TARGET_NODE for collection $COLLECTION" + exit 0 +fi + +# Get list of shards that have replicas on target node +shards_on_node=$(echo "$replicas_on_node" | cut -d: -f1 | sort -u) + +echo "Found replicas on $TARGET_NODE:" +echo "$replicas_on_node" +echo "" + +# ========================================= +# PHASE 1: Remove all replicas from node +# ========================================= +echo "=========================================" +echo "PHASE 1: Removing replicas from $TARGET_NODE" +echo "=========================================" + +# Get the directory of the current script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Call the delete-replicas subscript +"$SCRIPT_DIR/delete-replicas.sh" "$SOLR_URL" "$COLLECTION" "$TARGET_NODE" 2 + +echo "" +echo "All replicas removed from $TARGET_NODE" +echo "" + +# ========================================= +# PHASE 2: Add replicas back to node +# ========================================= +echo "=========================================" +echo "PHASE 2: Adding replicas back to $TARGET_NODE (async)" +echo "=========================================" + +async_ids=() +timestamp=$(date +%s) + +for shard in $shards_on_node; do + echo "Adding TLOG replica for $shard on $TARGET_NODE..." + + async_id="${COLLECTION}_${shard}_add_${timestamp}" + + # Delete any existing async status with this ID (ignore errors) + curl -s "$SOLR_URL/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null 2>&1 || true + + response=$(curl -s -w "\n%{http_code}" \ + "$SOLR_URL/admin/collections?action=ADDREPLICA&collection=$COLLECTION&shard=$shard&node=$TARGET_NODE&type=TLOG&async=$async_id") + + http_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | head -n -1) + + if [ "$http_code" != "200" ]; then + echo "Error: HTTP $http_code" + echo "$body" + exit 1 + fi + + async_ids+=("$async_id") + echo " Submitted (async id: $async_id)" +done + +echo "" +echo "Waiting for async operations to complete..." + +# Wait for all async operations to complete +for async_id in "${async_ids[@]}"; do + echo "Checking status of $async_id..." + + while true; do + status_response=$(curl -s "$SOLR_URL/admin/collections?action=REQUESTSTATUS&requestid=$async_id") + state=$(echo "$status_response" | jq -r '.status.state') + + if [ "$state" == "completed" ]; then + echo " $async_id: completed" + # Clean up the async request + curl -s "$SOLR_URL/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null + break + elif [ "$state" == "failed" ]; then + echo " $async_id: FAILED" + echo "$status_response" | jq '.status' + exit 1 + else + echo " $async_id: $state (waiting...)" + sleep 2 + fi + done +done + +echo "" +echo "=========================================" +echo "Replica cycling complete!" +echo "Collection: $COLLECTION" +echo "Node: $TARGET_NODE" +echo "Shards cycled: $(echo "$shards_on_node" | wc -w)" +echo "=========================================" + diff --git a/scripts/delete-replicas.sh b/scripts/delete-replicas.sh new file mode 100644 index 000000000000..965a002ca489 --- /dev/null +++ b/scripts/delete-replicas.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# ============================================================================= +# Script to remove all replicas from a node +# +# Usage: +# ./delete-replicas.sh [SOLR_URL] [COLLECTION] [TARGET_NODE] [TIMEOUT] +# +# Example: +# ./delete-replicas.sh http://localhost:8983/solr test solr2:8983_solr +# ./delete-replicas.sh http://localhost:8983/solr test solr2:8983_solr 10 +# ============================================================================= + +set -e + +SOLR_URL="${1:-http://localhost:8983/solr}" +COLLECTION="${2:-test}" +TARGET_NODE="${3:-solr2:8983_solr}" +TIMEOUT="${4:-10}" + +echo "Deleting replicas from $TARGET_NODE for collection $COLLECTION" +echo "Timeout: ${TIMEOUT}s" +echo "" + +# Get cluster status +cluster_status=$(curl -s "$SOLR_URL/admin/collections?action=CLUSTERSTATUS") + +# Find all replicas on the target node +# Format: shard_name:replica_name +replicas_on_node=$(echo "$cluster_status" | jq -r " + .cluster.collections[\"$COLLECTION\"].shards | to_entries[] | + .key as \$shard | + .value.replicas | to_entries[] | + select(.value.node_name == \"$TARGET_NODE\") | + \"\(\$shard):\(.key)\" +") + +if [ -z "$replicas_on_node" ]; then + echo "No replicas found on $TARGET_NODE for collection $COLLECTION" + exit 0 +fi + +echo "Found replicas on $TARGET_NODE:" +echo "$replicas_on_node" +echo "" + +echo "=========================================" +echo "Removing replicas from $TARGET_NODE" +echo "=========================================" + +deleted_count=0 +skipped_count=0 + +for replica_info in $replicas_on_node; do + shard=$(echo "$replica_info" | cut -d: -f1) + replica=$(echo "$replica_info" | cut -d: -f2) + + echo "Removing $replica from $shard..." + + response=$(curl -s -w "\n%{http_code}" --max-time "$TIMEOUT" \ + "$SOLR_URL/admin/collections?action=DELETEREPLICA&collection=$COLLECTION&shard=$shard&replica=$replica" 2>&1) || true + + http_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | head -n -1) + + # Check for timeout or empty response + if [[ -z "$http_code" ]] || [[ ! "$http_code" =~ ^[0-9]+$ ]]; then + echo " Warning: Request timed out for $replica, continuing..." + skipped_count=$((skipped_count + 1)) + continue + fi + + if [ "$http_code" != "200" ]; then + echo " Warning: HTTP $http_code for $replica" + echo " $body" + echo " Continuing with next replica..." + skipped_count=$((skipped_count + 1)) + continue + fi + + echo " Done" + deleted_count=$((deleted_count + 1)) +done + +echo "" +echo "=========================================" +echo "Replica deletion complete!" +echo "Collection: $COLLECTION" +echo "Node: $TARGET_NODE" +echo "Deleted: $deleted_count" +echo "Skipped: $skipped_count" +echo "=========================================" + diff --git a/scripts/ingest-docs-text-variants2.sh b/scripts/ingest-docs-text-variants2.sh new file mode 100644 index 000000000000..6b44315f27da --- /dev/null +++ b/scripts/ingest-docs-text-variants2.sh @@ -0,0 +1,341 @@ +#!/bin/bash +# +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# ============================================================================= +# Script to ingest documents with text field variants into a Solr collection +# +# Each document contains: +# - 5 text values shared across 3 field types: text1_s, text1_txt_en, text1_txt_sort (through text5_*) +# - 10 additional large filler _txt_en fields: filler1_txt_en through filler10_txt_en +# - Total: 25 text fields per document (15 shared + 10 filler) +# - Shared text sizes: uniform random between MIN-MAX (under 32KB DV limit) +# - Filler text sizes: uniform random between MIN_TXT_EN-MAX_TXT_EN (no DV limit, can be large) +# +# Usage: +# ./ingest-docs-text-variants.sh [SOLR_URL] [COLLECTION] [START_ID] [NUM_THREADS] \ +# [TOTAL_DOCS] [BATCH_SIZE] [MIN_TEXT_SIZE] [MAX_TEXT_SIZE] [MIN_TXT_EN_SIZE] [MAX_TXT_EN_SIZE] +# +# Parameters: +# SOLR_URL - Solr base URL (default: http://localhost:8983/solr) +# COLLECTION - Collection name (default: test) +# START_ID - Starting document ID (default: 0) +# NUM_THREADS - Number of parallel threads (default: 10) +# TOTAL_DOCS - Total documents to ingest (default: 10000000) +# BATCH_SIZE - Documents per batch (default: 100) +# MIN_TEXT_SIZE - Min text size for shared fields (default: 1024) +# MAX_TEXT_SIZE - Max text size for shared fields (default: 30720, under 32KB DV limit) +# MIN_TXT_EN_SIZE - Min text size for filler _txt_en fields (default: 1024) +# MAX_TXT_EN_SIZE - Max text size for filler _txt_en fields (default: 102400, ~100KB, no DV limit) +# ============================================================================= + +set -e + +# Ensure output is unbuffered and locale handles binary input safely +export BASH_XTRACEFD=2 +export LC_ALL=C + +SOLR_URL="${1:-http://localhost:8983/solr}" +COLLECTION="${2:-test}" +START_ID="${3:-0}" +NUM_THREADS="${4:-10}" +TOTAL_DOCS="${5:-10000000}" +# Reduced batch size due to larger document sizes (up to ~450KB per doc) +BATCH_SIZE="${6:-100}" + +# Text size range for _s and _txt_sort fields: 1KB to 30KB (under 32KB docValue limit) +MIN_TEXT_SIZE="${7:-1024}" +MAX_TEXT_SIZE="${8:-30720}" + +# Text size range for _txt_en fields: 1KB to 100KB (no docValue limit, stored only) +MIN_TXT_EN_SIZE="${9:-1024}" +MAX_TXT_EN_SIZE="${10:-102400}" + +# Pre-generated block size (max of both maximums) +FIELD_SIZE=$((MAX_TEXT_SIZE > MAX_TXT_EN_SIZE ? MAX_TEXT_SIZE : MAX_TXT_EN_SIZE)) + +END_ID=$((START_ID + TOTAL_DOCS)) +DOCS_PER_THREAD=$((TOTAL_DOCS / NUM_THREADS)) + +# Pre-calculate common values to avoid spawning 'date' per document +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +echo "Ingesting $TOTAL_DOCS documents into $SOLR_URL/$COLLECTION" +echo "Each document has 25 text fields:" +echo " - 15 shared fields (5 values × 3 types: _s, _txt_en, _txt_sort)" +echo " - 10 filler _txt_en fields (filler1_txt_en through filler10_txt_en)" +echo "Shared text sizes: ${MIN_TEXT_SIZE}-${MAX_TEXT_SIZE} bytes (under 32KB DV limit)" +echo "Filler text sizes: ${MIN_TXT_EN_SIZE}-${MAX_TXT_EN_SIZE} bytes (no DV limit)" +echo "Parallel threads: $NUM_THREADS" + +generate_text() { + local size=$1 + # Use redirection and /dev/urandom safely + tr -dc 'a-zA-Z0-9 ' < /dev/urandom | head -c "$size" +} +export -f generate_text + +# Generate random size between MIN and MAX (uniform distribution) +random_text_size() { + echo $((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE + 1))) +} +export -f random_text_size + +echo "Pre-generating random text blocks (30KB each for maximum coverage)..." +TEXT_BLOCK_1=$(generate_text $FIELD_SIZE) +echo " Generated block 1" +TEXT_BLOCK_2=$(generate_text $FIELD_SIZE) +echo " Generated block 2" +TEXT_BLOCK_3=$(generate_text $FIELD_SIZE) +echo " Generated block 3" +TEXT_BLOCK_4=$(generate_text $FIELD_SIZE) +echo " Generated block 4" +TEXT_BLOCK_5=$(generate_text $FIELD_SIZE) +echo " Generated block 5" +echo "Text blocks ready." + +# Function to create a single document JSON +# Generates 5 text values shared across all 3 field types (_s, _txt_en, _txt_sort) +# Plus 10 additional large filler _txt_en fields (no DV limit) +# Plus non-textual fields from original script +create_document() { + local id=$1 + + # Non-textual fields (from original script) + local r_int=$((RANDOM % 5)) + local r_dec=$((RANDOM % 100)) + local views=$((RANDOM * RANDOM % 1000000)) + + # Generate 5 text values shared across _s, _txt_en, and _txt_sort fields (under 32KB DV limit) + local size1=$((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE))) + local start1=$((RANDOM % (FIELD_SIZE - size1 + 1))) + local text1="${TEXT_BLOCK_1:$start1:$size1}" + + local size2=$((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE))) + local start2=$((RANDOM % (FIELD_SIZE - size2 + 1))) + local text2="${TEXT_BLOCK_2:$start2:$size2}" + + local size3=$((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE))) + local start3=$((RANDOM % (FIELD_SIZE - size3 + 1))) + local text3="${TEXT_BLOCK_3:$start3:$size3}" + + local size4=$((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE))) + local start4=$((RANDOM % (FIELD_SIZE - size4 + 1))) + local text4="${TEXT_BLOCK_4:$start4:$size4}" + + local size5=$((MIN_TEXT_SIZE + RANDOM % (MAX_TEXT_SIZE - MIN_TEXT_SIZE))) + local start5=$((RANDOM % (FIELD_SIZE - size5 + 1))) + local text5="${TEXT_BLOCK_5:$start5:$size5}" + + # Generate 10 large filler _txt_en fields (no docValues, can be much larger) + local filler_size1=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start1=$((RANDOM % (FIELD_SIZE - filler_size1 + 1))) + local filler1="${TEXT_BLOCK_1:$filler_start1:$filler_size1}" + + local filler_size2=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start2=$((RANDOM % (FIELD_SIZE - filler_size2 + 1))) + local filler2="${TEXT_BLOCK_2:$filler_start2:$filler_size2}" + + local filler_size3=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start3=$((RANDOM % (FIELD_SIZE - filler_size3 + 1))) + local filler3="${TEXT_BLOCK_3:$filler_start3:$filler_size3}" + + local filler_size4=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start4=$((RANDOM % (FIELD_SIZE - filler_size4 + 1))) + local filler4="${TEXT_BLOCK_4:$filler_start4:$filler_size4}" + + local filler_size5=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start5=$((RANDOM % (FIELD_SIZE - filler_size5 + 1))) + local filler5="${TEXT_BLOCK_5:$filler_start5:$filler_size5}" + + local filler_size6=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start6=$((RANDOM % (FIELD_SIZE - filler_size6 + 1))) + local filler6="${TEXT_BLOCK_1:$filler_start6:$filler_size6}" + + local filler_size7=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start7=$((RANDOM % (FIELD_SIZE - filler_size7 + 1))) + local filler7="${TEXT_BLOCK_2:$filler_start7:$filler_size7}" + + local filler_size8=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start8=$((RANDOM % (FIELD_SIZE - filler_size8 + 1))) + local filler8="${TEXT_BLOCK_3:$filler_start8:$filler_size8}" + + local filler_size9=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start9=$((RANDOM % (FIELD_SIZE - filler_size9 + 1))) + local filler9="${TEXT_BLOCK_4:$filler_start9:$filler_size9}" + + local filler_size10=$((MIN_TXT_EN_SIZE + RANDOM % (MAX_TXT_EN_SIZE - MIN_TXT_EN_SIZE))) + local filler_start10=$((RANDOM % (FIELD_SIZE - filler_size10 + 1))) + local filler10="${TEXT_BLOCK_5:$filler_start10:$filler_size10}" + + # Output document JSON + # All 3 field types (_s, _txt_en, _txt_sort) share the same text values + # Plus 10 large filler _txt_en fields + echo "{ + \"id\": \"doc_$id\", + \"text1_s\": \"$text1\", + \"text1_txt_en\": \"$text1\", + \"text1_txt_sort\": \"$text1\", + \"text2_s\": \"$text2\", + \"text2_txt_en\": \"$text2\", + \"text2_txt_sort\": \"$text2\", + \"text3_s\": \"$text3\", + \"text3_txt_en\": \"$text3\", + \"text3_txt_sort\": \"$text3\", + \"text4_s\": \"$text4\", + \"text4_txt_en\": \"$text4\", + \"text4_txt_sort\": \"$text4\", + \"text5_s\": \"$text5\", + \"text5_txt_en\": \"$text5\", + \"text5_txt_sort\": \"$text5\", + \"filler1_txt_en\": \"$filler1\", + \"filler2_txt_en\": \"$filler2\", + \"filler3_txt_en\": \"$filler3\", + \"filler4_txt_en\": \"$filler4\", + \"filler5_txt_en\": \"$filler5\", + \"filler6_txt_en\": \"$filler6\", + \"filler7_txt_en\": \"$filler7\", + \"filler8_txt_en\": \"$filler8\", + \"filler9_txt_en\": \"$filler9\", + \"filler10_txt_en\": \"$filler10\", + \"category_s\": \"category_$((id % 100))\", + \"author_s\": \"author_$((id % 500))\", + \"tags_ss\": [\"tag_$((id % 50))\", \"tag_$((id % 30))\", \"tag_$((id % 20))\"], + \"created_dt\": \"$TIMESTAMP\", + \"views_i\": $views, + \"rating_f\": $r_int.$r_dec +}" +} + +export -f create_document + +ingest_range() { + local thread_id=$1 + local range_start=$2 + local range_end=$3 + local thread_docs=$((range_end - range_start)) + local current_id=$range_start + local batch_num=0 + local docs_ingested=0 + local thread_start_time=$(date +%s) + + echo "[Thread $thread_id] Starting: $range_start to $((range_end - 1))" + + while [ $current_id -lt $range_end ]; do + # Generate fresh random blocks for this batch to ensure higher entropy + local TEXT_BLOCK_1=$(generate_text $FIELD_SIZE) + local TEXT_BLOCK_2=$(generate_text $FIELD_SIZE) + local TEXT_BLOCK_3=$(generate_text $FIELD_SIZE) + local TEXT_BLOCK_4=$(generate_text $FIELD_SIZE) + local TEXT_BLOCK_5=$(generate_text $FIELD_SIZE) + + batch_num=$((batch_num + 1)) + batch_start=$current_id + batch_end=$((current_id + BATCH_SIZE)) + + if [ $batch_end -gt $range_end ]; then + batch_end=$range_end + fi + + # Write batch JSON directly to temp file + temp_file=$(mktemp) + printf '[' > "$temp_file" + + # Loop optimization: minimize checks inside loop + create_document $batch_start >> "$temp_file" + for ((i=batch_start+1; i> "$temp_file" + create_document $i >> "$temp_file" + done + + printf ']' >> "$temp_file" + + # Send batch to Solr (added timeouts, increased max-time for larger payloads) + response=$(curl -s -S --connect-timeout 10 --max-time 600 -w "\n%{http_code}" -X POST \ + "$SOLR_URL/$COLLECTION/update?commit=false" \ + -H "Content-Type: application/json" \ + -d @"$temp_file") + + rm -f "$temp_file" + + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" != "200" ]; then + echo "[Thread $thread_id] Error: HTTP $http_code on batch $batch_num" + # Print error response safely + echo "$response" | head -n -1 + fi + + current_id=$batch_end + docs_ingested=$((current_id - range_start)) + + # Progress update every 10 batches (reduced from 20 due to larger docs) + if [ $((batch_num % 10)) -eq 0 ]; then + elapsed=$(($(date +%s) - thread_start_time)) + if [ $elapsed -gt 0 ]; then + rate=$((docs_ingested / elapsed)) + pct=$((docs_ingested * 100 / thread_docs)) + echo "[Thread $thread_id] Progress: $docs_ingested / $thread_docs ($pct%) - $rate docs/sec" + fi + fi + done + + echo "[Thread $thread_id] COMPLETED." +} + +export -f ingest_range +export SOLR_URL COLLECTION BATCH_SIZE TIMESTAMP TEXT_BLOCK_1 TEXT_BLOCK_2 TEXT_BLOCK_3 TEXT_BLOCK_4 TEXT_BLOCK_5 +export MIN_TEXT_SIZE MAX_TEXT_SIZE MIN_TXT_EN_SIZE MAX_TXT_EN_SIZE FIELD_SIZE + +start_time=$(date +%s) +pids=() + +echo "Starting $NUM_THREADS parallel threads..." + +for ((t=0; t&1 & + pids+=($!) +done + +echo "Waiting for threads..." +failed=0 +for pid in "${pids[@]}"; do + if ! wait $pid; then + failed=1 + fi +done + +if [ $failed -eq 1 ]; then + echo "One or more threads failed!" + exit 1 +fi + +echo "Committing..." +curl -s --connect-timeout 10 -X POST "$SOLR_URL/$COLLECTION/update?commit=true" -H "Content-Type: application/json" -d '{}' > /dev/null + +total_time=$(($(date +%s) - start_time)) +echo "Ingestion complete in ${total_time}s" + diff --git a/solr/benchmark/build-lib.sh b/solr/benchmark/build-lib.sh new file mode 100644 index 000000000000..a6d4c27b219f --- /dev/null +++ b/solr/benchmark/build-lib.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds all JARs and copies them to lib/ so jmh.sh can run without invoking gradle. + +set -e + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT="$SCRIPT_DIR/../.." +LIB_DIR="$SCRIPT_DIR/lib" + +echo "Building jars..." +"$REPO_ROOT/gradlew" -p "$REPO_ROOT" jar + +echo "Getting classpath from gradle..." +CLASSPATH=$("$REPO_ROOT/gradlew" -p "$SCRIPT_DIR" -q echoCp) + +echo "Copying JARs to lib/..." +rm -rf "$LIB_DIR" +mkdir -p "$LIB_DIR" + +# Copy all JAR dependencies +echo "$CLASSPATH" | tr ':' '\n' | while read -r jar; do + if [ -f "$jar" ] && [[ "$jar" == *.jar ]]; then + cp "$jar" "$LIB_DIR/" + fi +done + +# Copy the benchmark module's own JAR (echoCp outputs classes dir, not the JAR) +BENCHMARK_JAR=$(ls "$SCRIPT_DIR/build/libs"/solr-benchmark-*.jar 2>/dev/null | head -1) +if [ -n "$BENCHMARK_JAR" ]; then + cp "$BENCHMARK_JAR" "$LIB_DIR/" +fi + +JAR_COUNT=$(ls -1 "$LIB_DIR"/*.jar 2>/dev/null | wc -l) +echo "Done. Copied $JAR_COUNT JARs to lib/" +echo "You can now run jmh.sh without gradle being invoked." diff --git a/solr/benchmark/jmh.sh b/solr/benchmark/jmh.sh index 18f9875da192..6ef517f0da0b 100755 --- a/solr/benchmark/jmh.sh +++ b/solr/benchmark/jmh.sh @@ -54,8 +54,8 @@ echo "running JMH with args: $@" jvmArgs="-jvmArgs -Djmh.shutdownTimeout=5 -jvmArgs -Djmh.shutdownTimeout.step=3 -jvmArgs -Djava.security.egd=file:/dev/./urandom -jvmArgs -XX:+UnlockDiagnosticVMOptions -jvmArgs -XX:+DebugNonSafepoints -jvmArgs --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" gcArgs="-jvmArgs -XX:+UseG1GC -jvmArgs -XX:+ParallelRefProcEnabled" -# -jvmArgs -Dlog4j2.debug -loggingArgs="-jvmArgs -Dlog4jConfigurationFile=./log4j2-bench.xml -jvmArgs -Dlog4j2.is.webapp=false -jvmArgs -Dlog4j2.garbagefreeThreadContextMap=true -jvmArgs -Dlog4j2.enableDirectEncoders=true -jvmArgs -Dlog4j2.enable.threadlocals=true" +# -jvmArgs -Dlog4j2.debug +loggingArgs="-jvmArgs -Dlog4j2.configurationFile=./log4j2-bench.xml -jvmArgs -Dlog4j2.is.webapp=false -jvmArgs -Dlog4j2.garbagefreeThreadContextMap=true -jvmArgs -Dlog4j2.enableDirectEncoders=true -jvmArgs -Dlog4j2.enable.threadlocals=true -jvmArgs -Djava.util.logging.config.file=./logging.properties" #set -x diff --git a/solr/benchmark/log4j2-bench.xml b/solr/benchmark/log4j2-bench.xml index c3b81a84ca5e..ce7d42c23435 100644 --- a/solr/benchmark/log4j2-bench.xml +++ b/solr/benchmark/log4j2-bench.xml @@ -96,10 +96,18 @@ + + + + + + + + + - diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java index 4c4946a3ae86..9e5e714bb001 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java @@ -36,6 +36,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -79,7 +80,7 @@ public static class MiniClusterBenchState { public String zkHost; /** The Cluster. */ - MiniSolrCloudCluster cluster; + public MiniSolrCloudCluster cluster; /** The Client. */ public HttpJettySolrClient client; @@ -393,6 +394,7 @@ public void index(String collection, Docs docs, int docCount, boolean parallel) private void indexParallel(String collection, Docs docs, int docCount) throws InterruptedException { Meter meter = new Meter(); + AtomicReference indexingException = new AtomicReference<>(); ExecutorService executorService = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors(), @@ -429,6 +431,7 @@ public void run() { try { client.requestWithBaseUrl(url, updateRequest, collection); } catch (Exception e) { + indexingException.compareAndSet(null, e); throw new RuntimeException(e); } } @@ -444,6 +447,11 @@ public void run() { } scheduledExecutor.shutdown(); + + Exception ex = indexingException.get(); + if (ex != null) { + throw new RuntimeException("Indexing failed", ex); + } } private void indexBatch(String collection, Docs docs, int docCount, int batchSize) @@ -471,6 +479,106 @@ private void indexBatch(String collection, Docs docs, int docCount, int batchSiz log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s"); } + /** + * Index documents using multiple threads, each sending batches. + * + * @param collection the collection + * @param docs the docs generator + * @param docCount total number of docs to index + * @param numThreads number of parallel threads + * @param batchSize docs per batch/request + */ + @SuppressForbidden(reason = "This module does not need to deal with logging context") + public void indexParallelBatched( + String collection, Docs docs, int docCount, int numThreads, int batchSize) + throws InterruptedException { + Meter meter = new Meter(); + AtomicReference indexingException = new AtomicReference<>(); + + ExecutorService executorService = + Executors.newFixedThreadPool(numThreads, new SolrNamedThreadFactory("SolrJMH Indexer")); + + // Progress logging + ScheduledExecutorService scheduledExecutor = + Executors.newSingleThreadScheduledExecutor( + new SolrNamedThreadFactory("SolrJMH Indexer Progress")); + scheduledExecutor.scheduleAtFixedRate( + () -> { + if (meter.getCount() >= docCount) { + scheduledExecutor.shutdown(); + } else { + log( + meter.getCount() + + "/" + + docCount + + " docs at " + + (long) meter.getMeanRate() + + " doc/s"); + } + }, + 5, + 5, + TimeUnit.SECONDS); + + // Split work across threads + int docsPerThread = docCount / numThreads; + int remainder = docCount % numThreads; + + for (int t = 0; t < numThreads; t++) { + final int threadDocsCount = docsPerThread + (t < remainder ? 1 : 0); + + executorService.execute( + () -> { + List batch = new ArrayList<>(batchSize); + + for (int i = 0; i < threadDocsCount; i++) { + batch.add(docs.inputDocument()); + + if (batch.size() >= batchSize) { + sendBatch(collection, batch, indexingException); + meter.mark(batch.size()); + batch.clear(); + } + } + + // Send remaining docs + if (!batch.isEmpty()) { + sendBatch(collection, batch, indexingException); + meter.mark(batch.size()); + } + }); + } + + executorService.shutdown(); + boolean terminated = false; + while (!terminated) { + terminated = executorService.awaitTermination(10, TimeUnit.MINUTES); + } + scheduledExecutor.shutdown(); + + Exception ex = indexingException.get(); + if (ex != null) { + throw new RuntimeException("Indexing failed", ex); + } + + log(meter.getCount() + " docs indexed at " + (long) meter.getMeanRate() + " doc/s"); + } + + private void sendBatch( + String collection, + List batch, + AtomicReference indexingException) { + try { + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.add(batch); + // Use first node - simpler and avoids thread-safety issues with random node selection + client.requestWithBaseUrl(nodes.get(0), updateRequest, collection); + } catch (Exception e) { + indexingException.compareAndSet(null, e); + throw new RuntimeException(e); + } + } + /** * Wait for merges. * diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java index a9860763dbe7..ead834232d7d 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java @@ -64,7 +64,30 @@ public static class BenchState { @Param({"false", "true"}) boolean useHttp1; - private int docs = 1000; + @Param("3") + int nodeCount; + + @Param("3") + int numShards; + + @Param("1") + int numReplicas; + + @Param("1000") + int docCount; + + @Param("4") + int indexThreads; // 0 = sequential indexing + + @Param("500") + int batchSize; + + @Param("1024") + int docSizeBytes; // Target document size in bytes (approximate) + + @Param("3") + int numTextFields; // Number of textN_ts fields to generate + private String zkHost; private ModifiableSolrParams params; private StreamContext streamContext; @@ -73,24 +96,70 @@ public static class BenchState { @Setup(Level.Trial) public void setup(MiniClusterBenchState miniClusterState) throws Exception { - miniClusterState.startMiniCluster(3); - miniClusterState.createCollection(collection, 3, 1); - Docs docGen = - docs() - .field("id", integers().incrementing()) - .field("text2_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) - .field("text3_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) - .field("int1_i_dv", integers().all()); - miniClusterState.index(collection, docGen, docs); + miniClusterState.startMiniCluster(nodeCount); + miniClusterState.createCollection(collection, numShards, numReplicas); + + Docs docGen = createDocsWithTargetSize(docSizeBytes, numTextFields); + + if (indexThreads > 0) { + miniClusterState.indexParallelBatched( + collection, docGen, docCount, indexThreads, batchSize); + } else { + miniClusterState.index(collection, docGen, docCount, false); + } miniClusterState.waitForMerges(collection); zkHost = miniClusterState.zkHost; + // Build field list dynamically based on numTextFields + StringBuilder flBuilder = new StringBuilder("id"); + for (int i = 1; i <= numTextFields; i++) { + flBuilder.append(",text").append(i).append("_ts"); + } + flBuilder.append(",int1_i_dv"); + params = new ModifiableSolrParams(); params.set(CommonParams.Q, "*:*"); - params.set(CommonParams.FL, "id,text2_ts,text3_ts,int1_i_dv"); + params.set(CommonParams.FL, flBuilder.toString()); params.set(CommonParams.SORT, "id asc,int1_i_dv asc"); - params.set(CommonParams.ROWS, docs); + params.set(CommonParams.ROWS, docCount); + } + + /** + * Creates a Docs generator that produces documents with approximately the target size. + * + * @param targetSizeBytes target document size in bytes (approximate) + * @param numFields number of textN_ts fields to generate + * @return Docs generator configured for the target size + */ + private Docs createDocsWithTargetSize(int targetSizeBytes, int numFields) { + // Calculate how many characters per field to approximate target size + // Each character is ~1 byte in basic Latin alphabet + // Account for field overhead, id field, and int field + int baseOverhead = 100; // Approximate overhead for id, int field, and field names + int availableBytes = Math.max(100, targetSizeBytes - baseOverhead); + int bytesPerField = availableBytes / Math.max(1, numFields); + + // Use multi-value strings: multi(N) creates N strings joined by spaces + // Calculate words and word length to hit target + int wordsPerField = Math.max(1, bytesPerField / 50); // ~50 chars per word avg + int wordLength = Math.min(64, Math.max(10, bytesPerField / Math.max(1, wordsPerField))); + int minWordLength = Math.max(5, wordLength - 10); + int maxWordLength = wordLength + 10; + + Docs docGen = docs().field("id", integers().incrementing()); + + for (int i = 1; i <= numFields; i++) { + docGen.field( + "text" + i + "_ts", + strings() + .basicLatinAlphabet() + .multi(wordsPerField) + .ofLengthBetween(minWordLength, maxWordLength)); + } + + docGen.field("int1_i_dv", integers().all()); + return docGen; } @Setup(Level.Iteration) diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index e517aea59307..09ae5fa43997 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -43,6 +43,7 @@ + diff --git a/solr/server/etc/jetty-http.xml b/solr/server/etc/jetty-http.xml index 02f53991c5c1..cf43b2836c85 100644 --- a/solr/server/etc/jetty-http.xml +++ b/solr/server/etc/jetty-http.xml @@ -34,6 +34,15 @@ + + + + + + + + + diff --git a/solr/server/etc/jetty-https.xml b/solr/server/etc/jetty-https.xml index 4a74eb125063..d5bcae7f14f9 100644 --- a/solr/server/etc/jetty-https.xml +++ b/solr/server/etc/jetty-https.xml @@ -54,6 +54,15 @@ + + + + + + + + + diff --git a/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java b/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java index 072a2add953c..9ccafca1a7c3 100644 --- a/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java +++ b/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java @@ -81,6 +81,7 @@ import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.http.MultiPart; +import org.eclipse.jetty.http2.SimpleFlowControlStrategy; import org.eclipse.jetty.http2.client.HTTP2Client; import org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2; import org.eclipse.jetty.io.ClientConnector; @@ -110,6 +111,27 @@ public class HttpJettySolrClient extends HttpSolrClientBase { */ public static final String CLIENT_CUSTOMIZER_SYSPROP = "solr.solrj.http.jetty.customizer"; + /** + * A Java system property to set the initial HTTP/2 session receive window size (in bytes). Only + * applies when using HTTP/2 transport. + */ + public static final String HTTP2_SESSION_RECV_WINDOW_SYSPROP = + "solr.http2.initialSessionRecvWindow"; + + /** + * A Java system property to set the initial HTTP/2 stream receive window size (in bytes). Only + * applies when using HTTP/2 transport. + */ + public static final String HTTP2_STREAM_RECV_WINDOW_SYSPROP = + "solr.http2.initialStreamRecvWindow"; + + /** + * A Java system property to enable the simple flow control strategy for HTTP/2. When set to + * "true", uses {@link SimpleFlowControlStrategy} instead of the default buffering strategy. Only + * applies when using HTTP/2 transport. + */ + public static final String HTTP2_SIMPLE_FLOW_CONTROL_SYSPROP = "solr.http2.useSimpleFlowControl"; + public static final String REQ_PRINCIPAL_KEY = "solr-req-principal"; private static final String USER_AGENT = "Solr[" + MethodHandles.lookup().lookupClass().getName() + "] " + SolrVersion.LATEST_STRING; @@ -275,9 +297,9 @@ private HttpClient createHttpClient(Builder builder) { HttpClient httpClient; HttpClientTransport transport; if (builder.shouldUseHttp1_1()) { - if (log.isDebugEnabled()) { - log.debug("Create HttpJettySolrClient with HTTP/1.1 transport"); - } + log.info( + "Create HttpJettySolrClient with HTTP/1.1 transport (solr.http1={})", + System.getProperty("solr.http1")); transport = new HttpClientTransportOverHTTP(clientConnector); httpClient = new HttpClient(transport); @@ -285,11 +307,12 @@ private HttpClient createHttpClient(Builder builder) { httpClient.setMaxConnectionsPerDestination(builder.getMaxConnectionsPerHost()); } } else { - if (log.isDebugEnabled()) { - log.debug("Create HttpJettySolrClient with HTTP/2 transport"); - } + log.info( + "Create HttpJettySolrClient with HTTP/2 transport (solr.http1={})", + System.getProperty("solr.http1")); HTTP2Client http2client = new HTTP2Client(clientConnector); + configureHttp2FlowControl(http2client); transport = new HttpClientTransportOverHTTP2(http2client); httpClient = new HttpClient(transport); httpClient.setMaxConnectionsPerDestination(4); @@ -326,6 +349,31 @@ private HttpClient createHttpClient(Builder builder) { return httpClient; } + /** + * Configures HTTP/2 flow control settings on the HTTP2Client based on system properties. Only + * applies settings when the corresponding system property is explicitly set. + */ + private void configureHttp2FlowControl(HTTP2Client http2client) { + Integer sessionRecvWindow = + EnvUtils.getPropertyAsInteger(HTTP2_SESSION_RECV_WINDOW_SYSPROP, null); + if (sessionRecvWindow != null) { + http2client.setInitialSessionRecvWindow(sessionRecvWindow); + log.info("LKZ Set HTTP/2 initial session recv window to {} bytes", sessionRecvWindow); + } + + Integer streamRecvWindow = + EnvUtils.getPropertyAsInteger(HTTP2_STREAM_RECV_WINDOW_SYSPROP, null); + if (streamRecvWindow != null) { + http2client.setInitialStreamRecvWindow(streamRecvWindow); + log.info("Set HTTP/2 initial stream recv window to {} bytes", streamRecvWindow); + } + + if (EnvUtils.getPropertyAsBool(HTTP2_SIMPLE_FLOW_CONTROL_SYSPROP, false)) { + http2client.setFlowControlStrategyFactory(SimpleFlowControlStrategy::new); + log.error("Using simple HTTP/2 flow control strategy"); + } + } + private void setupProxy(Builder builder, HttpClient httpClient) { if (builder.getProxyHost() == null) { return; diff --git a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java index 918ef1247f8c..f94a133c31d1 100644 --- a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java +++ b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java @@ -29,6 +29,9 @@ import org.apache.solr.util.SolrJettyTestRule; import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP; import org.eclipse.jetty.ee10.servlet.ServletHolder; +import org.eclipse.jetty.http2.FlowControlStrategy; +import org.eclipse.jetty.http2.SimpleFlowControlStrategy; +import org.eclipse.jetty.http2.client.HTTP2Client; import org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2; import org.junit.ClassRule; @@ -49,6 +52,73 @@ public void testSystemPropertyFlag() { } } + public void testHttp2FlowControlSystemProperties() { + // Test with custom session and stream recv window sizes + System.setProperty( + HttpJettySolrClient.HTTP2_SESSION_RECV_WINDOW_SYSPROP, String.valueOf(4 * 1024 * 1024)); + System.setProperty( + HttpJettySolrClient.HTTP2_STREAM_RECV_WINDOW_SYSPROP, String.valueOf(2 * 1024 * 1024)); + System.setProperty(HttpJettySolrClient.HTTP2_SIMPLE_FLOW_CONTROL_SYSPROP, "true"); + + try (var client = new HttpJettySolrClient.Builder().build()) { + var transport = client.getHttpClient().getTransport(); + assertTrue("Expected HTTP/2 transport", transport instanceof HttpClientTransportOverHTTP2); + + HttpClientTransportOverHTTP2 http2Transport = (HttpClientTransportOverHTTP2) transport; + HTTP2Client http2Client = http2Transport.getHTTP2Client(); + + assertEquals( + "Session recv window should be set", + 4 * 1024 * 1024, + http2Client.getInitialSessionRecvWindow()); + assertEquals( + "Stream recv window should be set", + 2 * 1024 * 1024, + http2Client.getInitialStreamRecvWindow()); + + // Verify simple flow control strategy is used + FlowControlStrategy.Factory factory = http2Client.getFlowControlStrategyFactory(); + FlowControlStrategy strategy = factory.newFlowControlStrategy(); + assertTrue( + "Expected SimpleFlowControlStrategy", strategy instanceof SimpleFlowControlStrategy); + } finally { + System.clearProperty(HttpJettySolrClient.HTTP2_SESSION_RECV_WINDOW_SYSPROP); + System.clearProperty(HttpJettySolrClient.HTTP2_STREAM_RECV_WINDOW_SYSPROP); + System.clearProperty(HttpJettySolrClient.HTTP2_SIMPLE_FLOW_CONTROL_SYSPROP); + } + } + + @SuppressWarnings("try") // HTTP2Client is AutoCloseable but doesn't need closing when not started + public void testHttp2FlowControlDefaultsUnchangedWhenPropertiesNotSet() { + // Ensure no flow control properties are set + System.clearProperty(HttpJettySolrClient.HTTP2_SESSION_RECV_WINDOW_SYSPROP); + System.clearProperty(HttpJettySolrClient.HTTP2_STREAM_RECV_WINDOW_SYSPROP); + System.clearProperty(HttpJettySolrClient.HTTP2_SIMPLE_FLOW_CONTROL_SYSPROP); + + // Get default values from a fresh HTTP2Client for comparison + // Note: HTTP2Client doesn't need to be closed if never started + HTTP2Client defaultHttp2Client = new HTTP2Client(); + int defaultSessionWindow = defaultHttp2Client.getInitialSessionRecvWindow(); + int defaultStreamWindow = defaultHttp2Client.getInitialStreamRecvWindow(); + + try (var client = new HttpJettySolrClient.Builder().build()) { + var transport = client.getHttpClient().getTransport(); + assertTrue("Expected HTTP/2 transport", transport instanceof HttpClientTransportOverHTTP2); + + HttpClientTransportOverHTTP2 http2Transport = (HttpClientTransportOverHTTP2) transport; + HTTP2Client http2Client = http2Transport.getHTTP2Client(); + + assertEquals( + "Session recv window should remain at default", + defaultSessionWindow, + http2Client.getInitialSessionRecvWindow()); + assertEquals( + "Stream recv window should remain at default", + defaultStreamWindow, + http2Client.getInitialStreamRecvWindow()); + } + } + public void testConnectToOldNodesUsingHttp1() throws Exception { JettyConfig jettyConfig = diff --git a/stream-benchmark-results.md b/stream-benchmark-results.md new file mode 100644 index 000000000000..959150bd4ca4 --- /dev/null +++ b/stream-benchmark-results.md @@ -0,0 +1,71 @@ +# Reproducible /Stream Performance Results + +## Scenario: HTTP/2 with 1 shard + +**Result: Slow** + +```bash +./jmh.sh -p useHttp1=false -p nodeCount=2 -p numShards=1 -p numReplicas=2 -p docCount=10000 -p indexThreads=14 -p batchSize=500 -p docSizeBytes=10024 -p numTextFields=25 StreamingSearch +``` + +| Benchmark | batchSize | docCount | docSizeBytes | indexThreads | nodeCount | numReplicas | numShards | numTextFields | useHttp1 | Mode | Cnt | Score | Error | Units | +|-----------|-----------|----------|--------------|--------------|-----------|-------------|-----------|---------------|----------|------|-----|-------|-------|-------| +| StreamingSearch.stream | 500 | 10000 | 10024 | 14 | 2 | 2 | 1 | 25 | false | thrpt | 4 | 0.257 | ± 0.308 | ops/s | + +--- + +## Scenario: HTTP/2 with 12 shards + +**Result: Hangs indefinitely** + +```bash +./jmh.sh -p useHttp1=false -p nodeCount=2 -p numShards=12 -p numReplicas=2 -p docCount=10000 -p indexThreads=14 -p batchSize=500 -p docSizeBytes=10024 -p numTextFields=25 StreamingSearch +``` + +It appears server's flow control forcibly sets recvWindow to 0 on the offending client. There appear to be too many concurrent streams fighting for the same piece of "session window". I'm attaching flow-control-stall.log which gives a more detailed view of this response stall. + +--- + +## Scenario: HTTP/2 with 12 shards and LOWER initialStreamRecvWindow + +**Result: Slow** + +```bash +./jmh.sh -p useHttp1=false -p nodeCount=2 -p numShards=12 -p numReplicas=2 -p docCount=10000 -p indexThreads=14 -p batchSize=500 -p docSizeBytes=10024 -p numTextFields=25 -jvmArgs -Dsolr.http2.initialStreamRecvWindow=500000 StreamingSearch +``` + +| Benchmark | batchSize | docCount | docSizeBytes | indexThreads | nodeCount | numReplicas | numShards | numTextFields | useHttp1 | Mode | Cnt | Score | Error | Units | +|-----------|-----------|----------|--------------|--------------|-----------|-------------|-----------|---------------|----------|------|-----|-------|-------|-------| +| StreamingSearch.stream | 500 | 10000 | 10024 | 14 | 2 | 2 | 12 | 25 | false | thrpt | 4 | 0.344 | ± 0.027 | ops/s | + +The reason setting initialStreamRecvWindow lower helps avoid the stall is because each response gets sequestered to a smaller portion of the total session window pool and so each shard progresses consistently, albeit slowly. This result is still poor. + +--- + +## Scenario: HTTP/2 with 12 shards and HIGHER initialSessionRecvWindow + +**Result: Slow** + +```bash +./jmh.sh -p useHttp1=false -p nodeCount=2 -p numShards=12 -p numReplicas=2 -p docCount=10000 -p indexThreads=14 -p batchSize=500 -p docSizeBytes=10024 -p numTextFields=25 -jvmArgs -Dsolr.http2.initialStreamRecvWindow=8000000 -jvmArgs -Dsolr.http2.initialSessionRecvWindow=96000000 StreamingSearch +``` + +| Benchmark | batchSize | docCount | docSizeBytes | indexThreads | nodeCount | numReplicas | numShards | numTextFields | useHttp1 | Mode | Cnt | Score | Error | Units | +|-----------|-----------|----------|--------------|--------------|-----------|-------------|-----------|---------------|----------|------|-----|-------|-------|-------| +| StreamingSearch.stream | 500 | 10000 | 10024 | 14 | 2 | 2 | 12 | 25 | false | thrpt | 4 | 0.332 | ± 0.050 | ops/s | + +In other words, increasing the client-side response window doesn't help. + +--- + +## Scenario: HTTP/1 with 12 shards + +**Result: Fast** + +```bash +./jmh.sh -p useHttp1=true -p nodeCount=2 -p numShards=12 -p numReplicas=2 -p docCount=10000 -p indexThreads=14 -p batchSize=500 -p docSizeBytes=10024 -p numTextFields=25 -jvmArgs -Dsolr.http1=true StreamingSearch +``` + +| Benchmark | batchSize | docCount | docSizeBytes | indexThreads | nodeCount | numReplicas | numShards | numTextFields | useHttp1 | Mode | Cnt | Score | Error | Units | +|-----------|-----------|----------|--------------|--------------|-----------|-------------|-----------|---------------|----------|------|-----|-------|-------|-------| +| StreamingSearch.stream | 500 | 10000 | 10024 | 14 | 2 | 2 | 12 | 25 | true | thrpt | 4 | 2.301 | ± 0.676 | ops/s |