]> Wikimedia Canada | Git repositories - eccc_to_commons.git/commitdiff
Add beginning offset
authorPierre Choffet <peuc@wanadoo.fr>
Fri, 29 May 2020 15:22:35 +0000 (11:22 -0400)
committerPierre Choffet <peuc@wanadoo.fr>
Fri, 29 May 2020 15:22:35 +0000 (11:22 -0400)
mediawiki_post.sh

index bcddb358c9fa99d2f79b62f1864da693765e148a..3e4c3c3aa2bf59c2c8571777dfea1b8a6355ca55 100755 (executable)
@@ -21,7 +21,8 @@ set -o pipefail
 
 ENDPOINT='https://commons.wikimedia.org/w/api.php'
 NAMESPACE='Data'
-UPLOAD_MAX_RATE=15 # Per minute
+UPLOAD_MAX_RATE=15    # Per minute
+RESTART_AFTER_PATH='' # i.e. weather.gc.ca/Monthly/1013241.tab
 
 USERNAME_PATH='login_username'
 PASSWORD_PATH='login_password'
@@ -110,11 +111,34 @@ login
 requestCSRFToken > "${CSRF_TOKEN_PATH}"
 readCSRFToken
 
+# Prepare start offset
+if [ "${RESTART_AFTER_PATH}" == '' ]
+then
+       START_OFFSET_REACHED=1
+else
+       START_OFFSET_REACHED=0
+fi
+
 MINUTE=$(date +%M)
 MINUTE_UPLOADS=0
+
 while IFS= read -r -d '' -u 9
 do
+       # Prepare path
        URI_PATH=${NAMESPACE}:$(realpath --relative-to="${SOURCE}" "${REPLY}")
+
+       # Check start offset
+       if [ ${START_OFFSET_REACHED} -eq 0 ]
+       then
+               if [ ${URI_PATH} == "${NAMESPACE}:${RESTART_AFTER_PATH}" ]
+               then
+                       START_OFFSET_REACHED=1
+               fi
+
+               continue
+       fi
+
+       # Upload
        BODY=$(curl -X POST -d 'action=edit' --data-urlencode "title=${URI_PATH}" --data-urlencode "text@${REPLY}" -d "token=${CSRF_TOKEN}" -d 'format=xml' -b "${COOKIE_JAR}" -c "${COOKIE_JAR}" "${ENDPOINT}")
        RESULT=$(echo "${BODY}" | xmlstarlet sel -t -v '/api/edit/@result' -)