From 601a2c77318c4b989c6a4244e1e8eb35f1020e61 Mon Sep 17 00:00:00 2001 From: Pierre Choffet Date: Fri, 29 May 2020 11:22:35 -0400 Subject: [PATCH] Add beginning offset --- mediawiki_post.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/mediawiki_post.sh b/mediawiki_post.sh index bcddb35..3e4c3c3 100755 --- a/mediawiki_post.sh +++ b/mediawiki_post.sh @@ -21,7 +21,8 @@ set -o pipefail ENDPOINT='https://commons.wikimedia.org/w/api.php' NAMESPACE='Data' -UPLOAD_MAX_RATE=15 # Per minute +UPLOAD_MAX_RATE=15 # Per minute +RESTART_AFTER_PATH='' # i.e. weather.gc.ca/Monthly/1013241.tab USERNAME_PATH='login_username' PASSWORD_PATH='login_password' @@ -110,11 +111,34 @@ login requestCSRFToken > "${CSRF_TOKEN_PATH}" readCSRFToken +# Prepare start offset +if [ "${RESTART_AFTER_PATH}" == '' ] +then + START_OFFSET_REACHED=1 +else + START_OFFSET_REACHED=0 +fi + MINUTE=$(date +%M) MINUTE_UPLOADS=0 + while IFS= read -r -d '' -u 9 do + # Prepare path URI_PATH=${NAMESPACE}:$(realpath --relative-to="${SOURCE}" "${REPLY}") + + # Check start offset + if [ ${START_OFFSET_REACHED} -eq 0 ] + then + if [ ${URI_PATH} == "${NAMESPACE}:${RESTART_AFTER_PATH}" ] + then + START_OFFSET_REACHED=1 + fi + + continue + fi + + # Upload BODY=$(curl -X POST -d 'action=edit' --data-urlencode "title=${URI_PATH}" --data-urlencode "text@${REPLY}" -d "token=${CSRF_TOKEN}" -d 'format=xml' -b "${COOKIE_JAR}" -c "${COOKIE_JAR}" "${ENDPOINT}") RESULT=$(echo "${BODY}" | xmlstarlet sel -t -v '/api/edit/@result' -) -- 2.42.0