#!/bin/bash

# Script to list and download files from a public S3 bucket using curl and awk

# Authors: Henrik Virta and Seppo Hassinen, Finnish Meteorological Institute (henrik.virta@fmi.fi)
# Update: SH, added HCHO and SIF

#### SCRIPT FOR DOWNLOADING GRIDDED L3 TROPOMI FILES FROM KAT S3 SERVER
#
# Syntax: bash download_l3_s3tropomi.sh BUCKET TIMEPERIOD STARTTIME ENDTIME
# BUCKET: no2/o3/co/ch4/so2-(nrti-)l3
# TIMEPERIOD: daily/monthly/yearly
# STARTTIME: YYYY-MM-DD (inclusive)
# ENDTIME: YYYY-MM-DD (exclusive)
#
# Example usage: sh download_l3_s3tropomi.sh no2-l3 daily 2020-06-01 2020-07-01
#
# Requirements: Bash shell. Either GNU Linux system or BSD system (like Mac) with installed ggrep, awk and curl          

# Script makes use of GNU coreutils commands like "date" rather than their BSD counterparts.
# Such commands are by default prefixed with a "g" (e.g. "gdate") if installed via Homebrew,
# but can be set to override the BSD ones with a modified PATH.
if command -v gdate &> /dev/null; then
    export PATH="/usr/local/opt/coreutils/libexec/gnubin:${PATH}"
fi

# Interpret input parameters
s3bucket="$1"
timeperiod="$2"
startdate="$3"
enddate="$4"
bucket_url="https://tropomi-$s3bucket.data.lit.fmi.fi"
continuation_token=""


# Validate inputs (omitted for brevity, keep your existing validation logic here)
# Check that name of S3 bucket is correct
if [[ ! "$s3bucket" =~ ^(no2|o3|co|ch4|so2|hcho|sif)-(nrti-l3|l3)$ ]]; then
    echo "Incorrect S3 bucket name! Syntax: no2/co/o3/ch4/so2/hcho/sif-(nrti-)l3"
    exit 1
fi

# Check that time period is correct
if [[ ! "$timeperiod" =~ ^(daily|monthly|yearly)$ ]]; then
    echo "Incorrect time period name! Options: daily/monthly/yearly"
    exit 1
fi

# Check date syntaxes
dateregex='^20[0-9]{2}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'
if ! [[ "$startdate" =~ $dateregex && "$enddate" =~ $dateregex ]]; then
    echo "Incorrect date format! Syntax: 20YY-MM-DD"
    exit 1
fi

# Ensure end date is greater than start date
if [[ ! "$enddate" > "$startdate" ]]; then
    echo "End date must be greater than start date!"
    exit 1
fi

# Function to format dates according to the time period
format_dates() {
    case "$1" in
        daily)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
            echo $(date -d "$2" -u +"%Y%m%d")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y%m%d")
        fi
        ;;
        monthly)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
            echo $(date -d "$2" -u +"%Y%m")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y%m")
        fi
        ;;
        yearly)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
          echo $(date -d "$2" -u +"%Y")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y")
        fi
        ;;
    esac
}

# Format start and end dates
formatted_startdate=$(format_dates "$timeperiod" "$startdate")
formatted_enddate=$(format_dates "$timeperiod" "$enddate")

echo "Formatted startdate: $formatted_startdate"
echo "Formatted enddate: $formatted_enddate"

# Fetch bucket contents using curl
#curl -s "$bucket_url" | grep -oP '(?<=<Key>).*?(?=</Key>)' > bucket_contents.txt
temp_file="bucket_contents.txt"

# Try using grep with -P, and if it fails, fall back to a different command or strategy
if grep --version 2>&1 | grep -q BSD; then
    # System has BSD grep
    if which ggrep >/dev/null 2>&1; then
        grep_cmd="ggrep"
    else
        echo "GNU grep (ggrep) not found. Script requires Perl-compatible regular expressions (-P)."
        exit 1
    fi

else
    # GNU grep is available, for systems like macOS
    grep_cmd="grep"
fi


while : ; do
    response_file=$(mktemp)
    if [[ -n $continuation_token ]]; then
        curl -s "$bucket_url?list-type=2&continuation-token=$continuation_token" > "$response_file"
    else
        curl -s "$bucket_url?list-type=2" > "$response_file"
    fi

    # Extract keys and continuation token from response
    # This requires proper XML parsing to reliably extract keys and the token
      $grep_cmd -oP '(?<=<Key>).*?(?=</Key>)' "$response_file" >> "$temp_file"
      continuation_token=$($grep_cmd -oPm1 '(?<=<NextContinuationToken>).*?(?=</NextContinuationToken>)' "$response_file")

    # Break the loop if no continuation token is found
    if [[ -z $continuation_token ]]; then
        break
    fi

    # Cleanup temporary response file
    rm "$response_file"
done

# Filter matching files based on time period and date range
pattern=""
case "$timeperiod" in
    daily)
  if date --version >/dev/null 2>&1; then
    startdate=$(date -d "$startdate" -u +"%Y%m%d")
    enddate=$(date -d "$enddate" -u +"%Y%m%d")
  else
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y%m%d")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y%m%d")
  fi
  matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_dailycomposite_[0-9]{8}\.png$/ {
    date = $4
    sub(/^.*_dailycomposite_/, "", date)
    sub(/\.png$/, "", date)

    if (date >= FIRST && date < LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
    ;;
    monthly)
  if date --version >/dev/null 2>&1; then
    # GNU date (Linux)
    startdate=$(date -d "$startdate" -u +"%Y%m")
    enddate=$(date -d "$enddate" -u +"%Y%m")
  else # BSD date (macOS)
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y%m")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y%m")
  fi
matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_monthlycomposite_[0-9]{6}\.png$/ {
    date = $4
    sub(/^.*_monthlycomposite_/, "", date)
    sub(/\.png$/, "", date)

    month = substr(date, 1, 6)

    if (month >= FIRST && month <= LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
;;
    
    yearly)
  if date --version >/dev/null 2>&1; then
    startdate=$(date -d "$startdate" -u +"%Y")
    enddate=$(date -d "$enddate" -u +"%Y")
  else
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y")
  fi
 matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_yearlycomposite_[0-9]{4}\.png$/ {
    date = $4
    sub(/^.*_yearlycomposite_/, "", date)
    sub(/\.png$/, "", date)

    year = substr(date, 1, 4)

    if (year >= FIRST && year <= LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
esac


# Create destination folder
destfolder="S3_${s3bucket}"
mkdir -p "$destfolder"

# Download matched files
echo "$matches" | while read -r match; do
    echo "Downloading $match..."
    curl -s "$bucket_url/$match" -o "${destfolder}/$(basename "$match")"

    # Decompress if necessary, you may comment this out
    if [[ "$match" == *".png" ]]; then
        gzip -d "${destfolder}/$(basename "$match")"
    fi
done

rm bucket_contents.txt

echo "Finished!"