Add required scripts to generate almanac JSONs

diff --git a/commons_rules.xsd b/commons_rules.xsd
index 169dd3e066cdaa320049f1f44c91ff8bc5be48f0..49a50d31976575893aa56c5578143430ac779c09 100644 (file)
--- a/commons_rules.xsd
+++ b/commons_rules.xsd
@@ -3,7 +3,12 @@
      historical data prior of its import into Wikidata.
 
      All tests done in ECCC's bulkschema.xsd aren't replicated here as we're
      historical data prior of its import into Wikidata.
 
      All tests done in ECCC's bulkschema.xsd aren't replicated here as we're

-     focusing into pure Wikimedia Commons data quality -->
+     focusing into pure Wikimedia Commons data quality.
+
+     Schema is restrictive on purpose: up to date extreme values are hard coded
+     just like current year. It WILL break starting 2021 so we can make sure
+     someone has checked potential upstream structure change before running
+     Commons data update. -->

 <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
        <!-- Define elements -->
        <xsd:element name="climatedata">
 <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
        <!-- Define elements -->
        <xsd:element name="climatedata">

diff --git a/eccc_to_commons.sh b/eccc_to_commons.sh
index c580fbbc6518c99ba474e034e8887a7b9b7c1386..5d7b01071c177c6f81abaf4dfe7d4c5a85195f31 100755 (executable)
--- a/eccc_to_commons.sh
+++ b/eccc_to_commons.sh
@@ -92,8 +92,7 @@ do
        elif [ ${STATIONDATA_COUNT} -eq 0 ]&& \
             [ ${AL_MONTH_COUNT} -gt 0 ]
        then
        elif [ ${STATIONDATA_COUNT} -eq 0 ]&& \
             [ ${AL_MONTH_COUNT} -gt 0 ]
        then

-               echo "${REPLY}: Almanac data not compatible yet. Ignoring." >&2
-               continue
+               STYLESHEET_PATH='almanac_to_commons.xslt'

                DESTINATION_PATH="${DESTINATION}/weather.gc.ca/Almanac/${STATION_ID}.tab"
        else
                echo "${REPLY}: Cannot detect file type. Exiting."
                DESTINATION_PATH="${DESTINATION}/weather.gc.ca/Almanac/${STATION_ID}.tab"
        else
                echo "${REPLY}: Cannot detect file type. Exiting."


@@ -102,5 +101,5 @@ do
 
        echo "Processing ${REPLY} to ${DESTINATION_PATH}…" >&2
        mkdir -p "$(dirname "${DESTINATION_PATH}")"
 
        echo "Processing ${REPLY} to ${DESTINATION_PATH}…" >&2
        mkdir -p "$(dirname "${DESTINATION_PATH}")"

-       xmlstarlet tr "${STYLESHEET_PATH}" "${REPLY}" > "${DESTINATION_PATH}"
+       xmlstarlet tr "${STYLESHEET_PATH}" "${REPLY}" | jq > "${DESTINATION_PATH}"

 done 9< <( find "${SOURCE}" -type f -name '*.xml' -print0 )
 done 9< <( find "${SOURCE}" -type f -name '*.xml' -print0 )