Elasticsearch Yedekleme Politikası

path:
  repo:
    - /backups/my_backup_location

curl -X PUT "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_snapshot/apinizer-repository-<INDEX_KEY>?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/backups/my_backup_location",
 	"compress": true
  }
}
'

curl -X POST "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_snapshot/apinizer-repository-<INDEX_KEY>/_verify?pretty"

curl -X PUT "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_slm/policy/apinizer-slm-policy-<INDEX_KEY>?pretty" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 0 0 ? * 1#1 *", 
  "name": "<apinizer-snapshot-<INDEX_KEY>-{now/d}>", 
  "repository": "apinizer-repository-<INDEX_KEY>", 
  "config": { 
    "indices": ["apinizer-log-apiproxy-<INDEX_KEY>"],  
	"ignore_unavailable": false,
    "partial": false   
  },
  "retention": { 
    "expire_after": "30d", 
    "min_count": 5, 
    "max_count": 50 
  }
}
'

curl -X POST "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_slm/policy/apinizer-slm-policy-<INDEX_KEY>/_execute?pretty"

curl -X GET "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_snapshot/apinizer-repository-<INDEX_KEY>/apinizer-snapshot-<INDEX_KEY>*?pretty"

curl –XPUT "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_snapshot/apinizer-repository-<INDEX_KEY>/apinizer-snapshot-<INDEX_KEY>?wait_for_completion=true" -H 'Content-Type: application/json' –d 
'{
 "indices":"index001, index002, index003",    
 "ignore_unavailable":true, 
 "include_global_state": false
}'

curl -XPOST "http://<ELASTICSEARCH_IP_ADDRESS>:9200/_snapshot/apinizer-repository-<INDEX_KEY>/apinizer-snapshot-<INDEX_KEY>/_restore?pretty" -H 'Content-Type: application/json' -d 
'{
 "indices":"index00*",
 "ignore_unavailable":true,
 "include_global_state": false
}'

#!/bin/bash

#Logs will be written to a file
current_date=$(date +'%d-%m-%Y')
exec > logfile$current_date.log 2>&1

#Server IP's need to be set
es_snapshot_ip="<ELASTICSEARCH_IP_ADDRESS>"
es_restore_ip="<ELASTICSEARCH_BACKUP_SERVER>"
repository_dst_location="<BACKUP_PATH_REPO>"
log_key="<INDEX_KEY>"


time_start=`date +%s`

echo -e "\n\nScript has started on \"`date`\""

es_snapshot_address="http://$es_snapshot_ip:9200"
es_restore_address="http://$es_restore_ip:9200"
echo "Variables:"
echo " es_snapshot_address: $es_snapshot_address"
echo " es_restore_address: $es_restore_address"
echo " repository_dst_location: $repository_dst_location"


##Show repositories, take name and path
repository_name=$(curl -XGET -s "$es_snapshot_address/_snapshot/_all" |  jq -r 'keys[] | select(contains("repository"))')
repository_src_location=$(curl -XGET -s "$es_snapshot_address/_snapshot/_all" | jq -r ' .[].settings.location ' | head -1)
echo " repository_name: $repository_name"
echo " repository_src_location: $repository_src_location"

##Show snapshots on repository
echo -e "Command to be used: curl -XGET -s \"$es_snapshot_address/_snapshot/apinizer-repository-$log_key/_all\" | jq '.snapshots[].snapshot' | tr -d '\"' \n"
snapshot_name=$(curl -XGET -s "$es_snapshot_address/_snapshot/apinizer-repository-$log_key/_all" | jq '.snapshots[].snapshot' | tr -d '"')
echo " snapshot_name: $snapshot_name"
time_1=`date +%s`
echo -e "\nduration - since beginning: $((time_1-time_start)) seconds"

##Move Snapshot files to remote server
echo "---Moving Snapshot to remote server: Started"
size_snapshot=$(du -sh $repository_src_location)
echo "Snapshot file size: $size_snapshot"

size_dst_initial=$(ssh elasticsearch@$es_restore_ip "du -sh ${repository_dst_location/}")
echo "Target disk size before moving snapshot: $size_dst_initial"

echo "scp -r $repository_src_location/* elasticsearch@$es_restore_ip:$repository_dst_location/ &"
scp -r $repository_src_location/* elasticsearch@$es_restore_ip:$repository_dst_location/ &
SCP_PID=$!
wait $SCP_PID

echo "---Moving Snapshot to remote server: Done"
size_dst_afterscp=$(ssh elasticsearch@$es_restore_ip "du -sh ${repository_dst_location/}")

echo "Target disk size after moving snapshot: $size_dst_afterscp"

time_2=`date +%s`
echo "---duration - scp: $((time_2-time_1)) seconds"

if [ "$size_dst_initial" = "$size_dst_afterscp" ];
then
  echo "Moving snapshot file has failed. Script is being terminated."
  exit
fi



##Register repository on remote server
time_3=`date +%s`
echo -e "Command to be used: curl -XPUT \"$es_restore_address/_snapshot/$repository_name?pretty\" -H \"Content-Type: application/json\" -d '{   \"type\": \"fs\",   \"settings\": {     \"compress\" : \"true\",     \"location\": \"$repository_dst_location\"   } }' \n"
curl -XPUT "$es_restore_address/_snapshot/$repository_name?pretty" -H "Content-Type: application/json" -d '{
  "type": "fs",
  "settings": {
    "compress" : "true",
    "location": "'$repository_dst_location'"
  }
}'


##Start restoring snapshot
echo -e "\nRestore: Started"
echo "Command to be used: curl -XPOST -s \"$es_restore_address/_snapshot/$repository_name/$snapshot_name/_restore?pretty\" -H \"Content-Type: application/json\" -d '{   \"indices\": \".ds-apinizer-log-token-$log_key-*,.ds-apinizer-token-oauth-$log_key-*,.ds-apinizer-log-apiproxy-$log_key-*\",   \"rename_pattern\": \"(.ds-apinizer-)(.*$)\",   \"rename_replacement\": \"restored_$1$2\" }'"
index_to_close_list=()

while [ true ]
do
curl -XPOST -s "$es_restore_address/_snapshot/$repository_name/$snapshot_name/_restore?pretty" -H "Content-Type: application/json" -d '{
  "indices": ".ds-apinizer-log-token-$log_key-*,.ds-apinizer-log-apiproxy-$log_key-*",
  "rename_pattern": "(.ds-apinizer-)(.*$)",
  "rename_replacement": "restored_$1$2"
}' -o  restore.output

is_error_exist=$(grep -oPm 1 'error' < restore.output)
if [ "$is_error_exist" = "error" ];then
##There are always expected to be at least 1 conflicted index (The last one). Those indexes needs to be closed to write on them
	index_to_close=$(grep -oPm 1 'restored_.ds-apinizer-.*$log_key-\d{6}' < restore.output)
	curl -XPOST -s "$es_restore_address/$index_to_close/_close?pretty" >> closed_indexes.output
	index_to_close_list+=($index_to_close)
	echo "Conflicted index has closed: $index_to_close"
else
	echo "There is no conflicted indeks. Script is being continued."
	break
fi
done

##Check restore process hourly
echo -e "Command to be used: curl -XGET -s \"$es_restore_address/_cluster/state\" | jq '.restore.snapshots[].state' \n"
echo "Restore process will be checked every hour before continuing to script."
while [ true ]
do
sleep 3600
restore_result=$(curl -XGET -s "$es_restore_address/_cluster/state" | jq '.restore.snapshots[].state')


if [[ "$restore_result" = "STARTED" ] || [ "$restore_result" = "INIT" ]];then
	echo "Status of Restore process as per _cluster/state: $restore_result. Restore is in progress."
elif [ "$restore_result" = "DONE" ]; then
	echo "Status of Restore process as per _cluster/state: $restore_result. Continuing to script."
	break
elif [ "$restore_result" = "" ]; then
	echo "Status of Restore process could not obtained from _cluster/state. Continuing to script."
	break
fi
done


time_4=`date +%s`
echo "duration - restore: $((time_4-time_3)) seconds"

##Open closed indexes if there are any
for index in $index_to_close_list; do
  curl -XPOST -s "$es_restore_address/$index/_open"
done

##Setting visibility of restored indexes to visible
echo -e "\nSet visibility of restored indexes: Started"
cluster_dst_state=$(curl -s "$es_restore_address/_cluster/state")
restored_indices=$(echo "$cluster_dst_state" | jq '.metadata.indices | keys | .[]' | grep '^"restored_.*"')
restored_indices=${restored_indices//\"}

echo -e "Command to be used: curl -XPUT -s \"$es_restore_address/INDEX/_settings?pretty\" -H 'Content-Type: application/json' -d'{     \"index.hidden\": false   }' \n"
for index in $restored_indices; do
  curl -XPUT -s "$es_restore_address/$index/_settings?pretty" -H 'Content-Type: application/json' -d'{
    "index.hidden": false
  }'
done

echo "Set visibility of restored indexes: Done"
time_5=`date +%s`
echo "duration - restored visibility: $((time_5-time_4)) seconds"




##Making sure of if the restore process done. Index counts should be the same as snapshot file has
echo -e "\nChecking restore results: Started"
echo -e "Command to be used: curl -s \"$es_restore_address/_snapshot/$repository_name/$snapshot_name?pretty\" \n"
snapshot_json=$(curl -s "$es_restore_address/_snapshot/$repository_name/$snapshot_name?pretty")
snapshot_indices_array=$(echo "$snapshot_json" | jq -r '.snapshots[0].indices[]')
snapshot_index_count=$(echo "$snapshot_indices_array" | grep -c ".")
echo "Total number of indices in snapshot: $snapshot_index_count"

recovery_info=$(curl -s "$es_restore_address/_cat/recovery")
filtered_lines=$(echo "$recovery_info" | grep "$snapshot_name" | awk '$14 == "100.0%" || $4 == "100.0%"')

completed_count=$(echo "$filtered_lines" | grep -c "100.0%")
uncompleted_count=$(echo "$filtered_lines" | grep -cv "100.0%")

echo "-Restore completed: $completed_count"
echo "-Restore uncompleted: $uncompleted_count"

if [ "$uncompleted_count" -gt 0 ];
  then
	echo -e "\nUncompleted Indices:"
	echo "$filtered_lines" | grep -v "100.0%"
	echo -e "\n\n---There are indexes that could not be restored!---\n\n"
  elif [ "$uncompleted_count" -e 0 ];
  then
	echo -e "\n\n---Restore was successful."
	echo "Snapshot file will be deleted by Elasticsearch according to SLM policy."
    echo -e "To manually delete, following command can be used: curl -XDELETE \"$es_snapshot_address/_snapshot/$repository_name/$snapshot_name\" \n"
  else
	echo "Checking restore results has failed. Please check results manually."
fi
echo "Checking restore results: Done"
time_6=`date +%s`

echo "duration - checking restore results: $((time_6-time_5)) seconds"


time_end=`date +%s`
echo "\nduration - total time of script: $((time_end-time_start)) seconds"

##Clear the variables set to shell just in case
unset time_1 time_2 time_3 time_4 time_5 time_6 time_start time_end snapshot_json snapshot_indices_array snapshot_index_count recovery_info filtered_lines completed_count uncompleted_count current_date es_snapshot_address es_restore_address repository_dst_location repository_name repository_src_location snapshot_name cluster_dst_state restored_indices size_dst_afterscp size_dst_initial size_snapshot restore_result es_snapshot_ip es_restore_ip is_error_exist index_to_close_list index_to_close apinizer_adres
echo "Used variables has been cleansed."



echo -e "\n\nScript is done on \"`date`\" \n"

echo "Note: If there is a error log like -All shards failed-, those indexes needs to be deleted from remote cluster and restore process needs to be initialized partially."
##Script Ends##

chmod +x /path/to/ESMoveSnapshotAndRestore.sh
./path/to/ESMoveSnapshotAndRestore.sh &

crontab -e

0 23 3 * * /path/to/ESMoveSnapshotAndRestore.sh