From: Pierre Choffet <peuc@wanadoo.fr>
Date: Wed, 27 May 2020 17:46:50 +0000 (-0400)
Subject: Less naive rate limiter
X-Git-Url: https://git.wikimedia.ca/?a=commitdiff_plain;ds=sidebyside;h=77f4ccd72870f62719062e47eb83fa182489758b;hp=57211d6bfbfdf1992686423e288ae6524b4c4e5d;p=eccc_to_commons.git

Less naive rate limiter
---

diff --git a/mediawiki_post.sh b/mediawiki_post.sh
index 7c82687..bcddb35 100755
--- a/mediawiki_post.sh
+++ b/mediawiki_post.sh
@@ -21,6 +21,7 @@ set -o pipefail
 
 ENDPOINT='https://commons.wikimedia.org/w/api.php'
 NAMESPACE='Data'
+UPLOAD_MAX_RATE=15 # Per minute
 
 USERNAME_PATH='login_username'
 PASSWORD_PATH='login_password'
@@ -109,6 +110,8 @@ login
 requestCSRFToken > "${CSRF_TOKEN_PATH}"
 readCSRFToken
 
+MINUTE=$(date +%M)
+MINUTE_UPLOADS=0
 while IFS= read -r -d '' -u 9
 do
 	URI_PATH=${NAMESPACE}:$(realpath --relative-to="${SOURCE}" "${REPLY}")
@@ -117,8 +120,22 @@ do
 
 	case "${RESULT}" in
 	Success)
+		SECOND_NOW=$(date +%S)
+		MINUTE_NOW=$(date +%M)
+		MINUTE_UPLOADS=$((MINUTE_UPLOADS+1))
+
+		if [ ${MINUTE_NOW} -ne ${MINUTE} ]
+		then
+			MINUTE=$(date +%M)
+			MINUTE_UPLOADS=0
+		elif [ ${MINUTE_NOW} -eq ${MINUTE} ]&&[ ${MINUTE_UPLOADS} -eq ${UPLOAD_MAX_RATE} ]
+		then
+			sleep $((60-SECOND_NOW))
+			MINUTE=$(date +%M)
+			MINUTE_UPLOADS=0
+		fi
+
 		echo "Everything went right. Continue…"
-		sleep 4
 		;;
 	*)
 		echo "Unknown code: ${RESULT}. Exiting."