X-Git-Url: https://git.wikimedia.ca/?p=eccc_to_commons.git;a=blobdiff_plain;f=mediawiki_post.sh;h=3e4c3c3aa2bf59c2c8571777dfea1b8a6355ca55;hp=0e21026413b3cc357f94bdf3a7939b1b1cef9c24;hb=601a2c77318c4b989c6a4244e1e8eb35f1020e61;hpb=700372c3cf8433ed8e4fd643642f4545b14e6f21

diff --git a/mediawiki_post.sh b/mediawiki_post.sh
index 0e21026..3e4c3c3 100755
--- a/mediawiki_post.sh
+++ b/mediawiki_post.sh
@@ -19,9 +19,10 @@
 set -ex
 set -o pipefail
 
-#ENDPOINT='https://commons.wikimedia.org/w/api.php'
-ENDPOINT='http://mediawiki.banquise.tf/api.php'
+ENDPOINT='https://commons.wikimedia.org/w/api.php'
 NAMESPACE='Data'
+UPLOAD_MAX_RATE=15    # Per minute
+RESTART_AFTER_PATH='' # i.e. weather.gc.ca/Monthly/1013241.tab
 
 USERNAME_PATH='login_username'
 PASSWORD_PATH='login_password'
@@ -110,14 +111,54 @@ login
 requestCSRFToken > "${CSRF_TOKEN_PATH}"
 readCSRFToken
 
+# Prepare start offset
+if [ "${RESTART_AFTER_PATH}" == '' ]
+then
+	START_OFFSET_REACHED=1
+else
+	START_OFFSET_REACHED=0
+fi
+
+MINUTE=$(date +%M)
+MINUTE_UPLOADS=0
+
 while IFS= read -r -d '' -u 9
 do
+	# Prepare path
 	URI_PATH=${NAMESPACE}:$(realpath --relative-to="${SOURCE}" "${REPLY}")
+
+	# Check start offset
+	if [ ${START_OFFSET_REACHED} -eq 0 ]
+	then
+		if [ ${URI_PATH} == "${NAMESPACE}:${RESTART_AFTER_PATH}" ]
+		then
+			START_OFFSET_REACHED=1
+		fi
+
+		continue
+	fi
+
+	# Upload
 	BODY=$(curl -X POST -d 'action=edit' --data-urlencode "title=${URI_PATH}" --data-urlencode "text@${REPLY}" -d "token=${CSRF_TOKEN}" -d 'format=xml' -b "${COOKIE_JAR}" -c "${COOKIE_JAR}" "${ENDPOINT}")
 	RESULT=$(echo "${BODY}" | xmlstarlet sel -t -v '/api/edit/@result' -)
 
 	case "${RESULT}" in
 	Success)
+		SECOND_NOW=$(date +%S)
+		MINUTE_NOW=$(date +%M)
+		MINUTE_UPLOADS=$((MINUTE_UPLOADS+1))
+
+		if [ ${MINUTE_NOW} -ne ${MINUTE} ]
+		then
+			MINUTE=$(date +%M)
+			MINUTE_UPLOADS=0
+		elif [ ${MINUTE_NOW} -eq ${MINUTE} ]&&[ ${MINUTE_UPLOADS} -eq ${UPLOAD_MAX_RATE} ]
+		then
+			sleep $((60-SECOND_NOW))
+			MINUTE=$(date +%M)
+			MINUTE_UPLOADS=0
+		fi
+
 		echo "Everything went right. Continue…"
 		;;
 	*)