Tools/Gen_litedram: Major improvements and cleanup including work in ensuring it is reproducible

system/Makefile

@@ -112,7 +112,7 @@ nextpnr-gui: synth_ecp5.json
 
 external_ip/litedram_core_ecp5_phy.v:
 	${QUIET_DOWNLOAD}
-	${Q}../tools/gen_litedram.sh "$@"
+	${Q}../tools/gen_litedram.sh -q "$@"
 
 upload: upload_orangecrab
 

tools/gen_litedram.sh

@@ -3,59 +3,139 @@ set -eu
 
 SOURCE_CODE_URL=https://github.com/enjoy-digital/litedram/archive/refs/tags/2023.08.tar.gz
 SOURCE_MD5_SUM=afc2208c08e60994d126c445574960da
+PYTHON_APPIMAGE_URL=https://github.com/niess/python-appimage/releases/download/python3.9/python3.9.18-cp39-cp39-manylinux_2_28_x86_64.AppImage
+PYTHON_APPIMAGE_SUM=8c383ade3ace416cf508d5f458b30149
 
-if [ "$#" = 0 ] || echo "$1" | grep '^-' > /dev/null
+OUTPUT_ECP5_1_WISHBONE_MD5=c775ddb23fd97211f3ace40d6e8c2a5f
-then
+
-	echo This scripts generates a verilog file from the Migen project litedram
+
+help(){
+	echo This scripts generates a verilog file from the litedram project
 	echo It is currently hardcoded for the ECP5 PHY
-	echo The provided file system/external_ip/litedram_core_ecp5_phy.v is the output of this program *BUT MODIFIED*
 	echo
 	echo Usage:
 	echo \ \ \ "$0" \<output verilog file\>
+}
 
-	if ! [ "$#" = 0 ]
+STOP_PARSING=0
+OUT_FILE=""
+QUIET=0
+FORCE=0
+while [ $# -gt 0 ]; do
+	if [ "$STOP_PARSING" = "1" ]
 	then
-		if ! [ "$1" = "--help" ] && ! [ "$1" = "-h" ]
+		echo arguments after --
-		then
+		help
-			exit 1
-		else
-			exit 0
-		fi
-	else
 		exit 1
 	fi
+	case $1 in
+		-q|--quiet)
+			QUIET="$1"
+			shift
+			;;
+		-h|--help)
+			help
+			exit 0
+			;;
+		-f|--force)
+			FORCE=1
+			shift
+			;;
+		--)
+			if ! [ $# -gt 1 ]
+			then
+				echo No file after --
+				help
+				exit 1
+			fi
+			if [ "$OUT_FILE" ]
+			then
+				echo Error two output files given
+				exit 1
+			fi
+			OUT_FILE=$2
+			STOP_PARSING=1
+			shift
+			shift
+			;;
+		-*)
+			echo "Unknown option $1"
+			help
+			exit 1
+			;;
+		*)
+			if [ "$OUT_FILE" ]
+			then
+				echo Error two output files given
+				exit 1
+			fi
+			OUT_FILE=$1
+			shift # past argument
+			;;
+	esac
+done
+
+if [ "$OUT_FILE" = "" ]
+then
+	echo Error: No output file provided
+	help
+	exit 1
 fi
 
 TEMP_DIR=$(mktemp -d)
 trap 'rm -rf -- "$TEMP_DIR"' EXIT
 CUR_DIR=$(pwd)
 
-if echo "$1" | grep '^/' > /dev/null
+if ! echo "$OUT_FILE" | grep '^/' > /dev/null
 then
-	OUT_FILE=$1
+	OUT_FILE="${CUR_DIR}/${OUT_FILE}"
-else
-	OUT_FILE="${CUR_DIR}/$1"
 fi
 
 cd "$TEMP_DIR"
-#echo Downloading source code...
 
+if [ "$QUIET" = 0 ];then
+	echo Downloading litedram source code...
+fi
 wget --output-document=source.tar.gz --quiet "$SOURCE_CODE_URL"
 
+
 if [ "$(md5sum source.tar.gz)" != "${SOURCE_MD5_SUM}  source.tar.gz" ]
 then
-	#echo source doesn\'t match md5 sum
+	echo Downlaoded litedram source downloaded doesn\'t match md5 sum
 	exit 1
 fi
 
-#echo extracting the source...
+if [ "$QUIET" = 0 ];then
+	echo Downloading python appimage...
+fi
+wget --output-document=python.AppImage --quiet "$PYTHON_APPIMAGE_URL"
+
+if [ "$(md5sum python.AppImage)" != "${PYTHON_APPIMAGE_SUM}  python.AppImage" ]
+then
+	echo Downloaded python appimage source doesn\'t match md5 sum
+	exit 1
+fi
+
+chmod a+x python.AppImage
+
+./python.AppImage --appimage-extract >/dev/null
+
+PYTHON=$(realpath squashfs-root/AppRun)
+
+mkdir python_modules
+PYTHON_MODULES=$(realpath python_modules)
+
+if [ "$QUIET" = 0 ];then
+	echo extracting the source...
+fi
 tar xvf source.tar.gz > source_dir
 
 SOURCE_DIR=$(head -n 1 source_dir)
 rm source_dir
 
-#echo patching the source...
+if [ "$QUIET" = 0 ];then
-
+	echo patching the source...
+fi
 #Generate the following text by running this: (diff -Naur /dev/null examples/orangecrab_9086.yml | gzip -9 | base64 | tr -d '\n';echo)
 echo H4sIAAAAAAACA61US2+bQBA+179ihC+JMA4Pg4mlSo7sOLFaJ5bjHKqqQguMYxRg6QJ2UJX/3lkwUeKop3oOy2pmvm/niaZpcBHi7iIt4/iLqZuWZpiaboJhj6zhaGD13aFOYthDUOWlo6oq4AtLshjzCy5Y+oSBYL53qbtOv0rekVhguCNzONKtvnM5HNqubtoHkvEYNL2ng2r0LAPG4476p6MCdOEGUxQsBu2/RfIplFkUoDKi6/fZtf2omfZMcxc3pmtPlB69N1veXEHj1a8RQVZKdxLljqdITh+kC5PlIxRVhnCGL7DDFxHlwa4HOQo6JeS85kkwkV7EpUynK+szz3R1taiJOmoTauLle5YpQIi1KOnlJp4k9EKMWSUNJLqMeg5ljsAg5nvYsbhEKDiwHY9CEMhCiHGHcZQ+NdRdWN7+OEFF36qah4IlXsLDMsamWMpiPTC+OYOF4chMu/BQp9e45MA3UGwRfM5ECFzAw702nS8Wn8i81Cc+s3dUqrsy8VFIFr8qEJ4EL7P8HZiG8LmBAhj/Bku397BsWx1aTarrydKmRlGlmgTWssUEqgeEtG0pZwJ/l5gGFZyklFGalYUXxM/ehojrcAYuOnWTpQmCmAfPsHl79efkfrW6nqx/NYlU+UcwgDOQcNmCKi8w+URwRq2RGPgKB/T5EWmURkchgdmSzsl2TNmWZsIFwummTE6+X242KGgBsmJLkRhO77A8UtGOVcCThKUhNM5tNI+0kbDkosjhFNHQwgkvk3QUR/2zIuU+yrc+rbyny+08aEnfbn5rb3a5NoWsYN4+Cut8LPNgeK2/8nzt/AU14UE3kwUAAA== | base64 -d | gzip -d > 00-add_orangecrab_config.patch
 
@@ -64,7 +144,16 @@ cd "$SOURCE_DIR"
 patch -s -p0 < ../00-add_orangecrab_config.patch > /dev/null
 
 
-#echo building the file...
+if [ "$QUIET" = 0 ];then
+	echo Downloading required python modules...
+fi
+
+if ! "$PYTHON" -m pip install --target "$PYTHON_MODULES" install "pyaml==23.9.7" "migen==0.9.2" "litex==2023.08" "litedram==2023.8" > "$TEMP_DIR/build_log" 2> "$TEMP_DIR/build_log"
+then
+	echo ERROR: Failed download python modules
+	cat "$TEMP_DIR/build_log"
+	exit 1
+fi
 
 if ! ./setup.py build  > "$TEMP_DIR/build_log" 2> "$TEMP_DIR/build_log"
 then
@@ -75,18 +164,33 @@ fi
 
 cd build/lib/litedram/
 
-if ! python3 gen.py ../../../examples/orangecrab_9086.yml --no-compile-software  > "$TEMP_DIR/build_log" 2> "$TEMP_DIR/build_log"
+if ! PYTHONPATH=$PYTHON_MODULES "$PYTHON"  gen.py ../../../examples/orangecrab_9086.yml --no-compile-software  > "$TEMP_DIR/build_log" 2> "$TEMP_DIR/build_log"
 then
 	echo ERROR: Failed to run gen.py
 	cat "$TEMP_DIR/build_log"
 	exit 1
 fi
 
-#echo patching source file...
+if [ "$QUIET" = 0 ];then
-
+	echo patching source file...
+fi
 #This is just for some yosys warnings...
 echo H4sIAAAAAAACA82T326CMBTG73mKc7XBECnKAHUkvMeyNEdatZE/hiLEzPnsa5nL4pYh3izrTduc3/d9Jyet4ziw3IuMuWuseYsVdzNRc1ZhTtOy4uMGJmQydTzPmYRAgrkfzEk4noYzEgZh5INN1DJs2waX8cYt9lkG3yWRUo1nj5FPvMALzpIkASci/nQUgt3tASSJASilWBfw+tmFboKylaA7j7YVwxpH8LNGzrU3iC+roqh5tcKU01bXF/8igOYot30pHaCjzFNfGG25tTCcvw4EzFo8SEjMBwuWfC0KA/S6UKabspRqyxltMBNMwlMM0T0jiyHwM3nRvHlBLbHY5phuRMHJFwt3YJpXwB0eshIZFVJfleK39BaLWh+k1dmehhpXKLXgNIxOUVrWDbyauuKP/V1jWotG/WFlbRzhholUHBnEcb+7hj6GMti3rRR51bijVM+Weljv0yEXHY0EAAA= | base64 -d | gzip -d | patch -s -p0
 
+if ! [ "$(sed 's@//.*@@' build/gateware/litedram_core.v | md5sum )" = "$OUTPUT_ECP5_1_WISHBONE_MD5  -" ]
+then
+	if [ "$FORCE" = 0 ]
+	then
+		echo ERROR: File was successfully bult but the md5sum doesn\'t match the one used in development.
+		echo        You can ignore this error by passing the --force flag
+		exit 1
+	else
+		echo WARNING: File was successfully bult but the md5sum doesn\'t match the one used in development.
+	fi
+fi
+
 cp build/gateware/litedram_core.v "$OUT_FILE"
 
-#echo File built correctly!
+if [ "$QUIET" = 0 ];then
+	echo File built correctly!
+fi