#!/bin/sh
#
# cook-pkgdb - SliTaz cook: rebuild package databases for a packages
# directory plus the wok-wide split.db and maint.db.
#
# Produces in $PKGS: packages.list, packages.md5, packages.info,
# packages.equiv, descriptions.txt, bdeps.txt, files-list.lzma,
# bundle.tar.lzma. Produces in $cache: split.db, maint.db.
#
# Usage:
#   cook-pkgdb              # full rebuild (wok DBs + $PKGS lists)
#   cook-pkgdb <dir>        # full rebuild against packages dir <dir>
#   cook-pkgdb --flavors    # full rebuild + regenerate TazLiTo flavors
#   cook-pkgdb --rmpkg      # full rebuild + remove unused tazpkg files
#   cook-pkgdb --splitdb    # only refresh $cache/split.db
#   cook-pkgdb --maintdb    # only refresh $cache/maint.db
#
# Copyright (C) SliTaz GNU/Linux - GNU GPL v3
#

. /usr/lib/slitaz/libcook.sh


#
# Functions
#

usage() {
	cat <<EOT

$(boldify "$(_ 'Usage:')") cook-pkgdb [command|<packages-dir>]

$(boldify "$(_ 'Commands:')")
  usage|help|--help|-h $(_ 'Display this short usage.')
  --splitdb            $(_ 'Refresh $cache/split.db only.')
  --maintdb            $(_ 'Refresh $cache/maint.db only.')
  --flavors            $(_ 'Full rebuild + regenerate TazLiTo flavors.')
  --rmpkg              $(_ 'Full rebuild + remove unused tazpkg files.')

$(boldify "$(_ 'Default:')")
  cook-pkgdb           $(_ 'Refresh split.db + maint.db + lists in $PKGS.')
  cook-pkgdb <dir>     $(_ 'Same, against the packages dir <dir>.')

$(boldify "$(_ 'cook.conf:')")
  MAKE_BUNDLE="yes"    $(_ 'Also build bundle.tar.lzma (build host only).')

EOT
	exit 0
}


dblog() { tee -a $LOGS/pkgdb.log; }


# Human-readable file size (vs `du`, which rounds to block size).

filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }


# Rebuild $cache/split.db: one line per receipt mapping package name
# to its all_names() list (main + SPLIT).

recreate_split_db() {
	local db="$cache/split.db"

	cd $WOK
	for pkg in *; do
		[ -f "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE SPLIT
		. $WOK/$pkg/receipt
		echo -e "$PACKAGE\t$(all_names)"
	done > $db
}


# Rebuild $cache/maint.db: one line per receipt mapping maintainer
# email to package name, sorted.

recreate_maint_db() {
	local db="$cache/maint.db"

	cd $WOK
	for pkg in *; do
		[ -f "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE MAINTAINER
		. $WOK/$pkg/receipt
		# Strip "Name <" prefix and "> ..." suffix without forking sed:
		# keep only the address between the angle brackets.
		MAINTAINER=${MAINTAINER##*<}; MAINTAINER=${MAINTAINER%%>*}
		echo -e "$MAINTAINER\t$PACKAGE"
	done | sort > $db
}


# all_names: list main pkg + every name from SPLIT, with v1/v2 handling.
# Copied verbatim from cook so this tool is self-contained.

all_names() {
	# Get package names from $SPLIT variable. Skip the awk|tr fork when
	# there's no SPLIT (the common case) -- this runs ~6000x per pass.
	local split='' split_space first
	if [ -n "$SPLIT" ]; then
		split=$(echo -n $SPLIT \
			| awk '
				BEGIN { RS = " "; FS = ":"; }
				{ print $1; }' \
			| tr '\n' ' ')
	fi
	split_space=" $split "
	# Receipt format version sits on the first line; read it with the
	# shell instead of `head ... | fgrep` (2 forks/receipt -> 50x faster).
	read -r first < "$WOK/$pkg/receipt" 2>/dev/null
	case "$first" in
		*v2*) : ;;	# v2: honour $SPLIT ordering below
		*)
			# For receipts v1: $SPLIT may present in the $WANTED package,
			# but split packages have their own receipts
			echo $PACKAGE; return ;;
	esac
	if [ "${split_space/ $PACKAGE /}" != "$split_space" ]; then
		# $PACKAGE included somewhere in $SPLIT (probably in the end).
		# We should build packages in the order defined in the $SPLIT.
		echo $split
	else
		# We'll build the $PACKAGE, then all defined in the $SPLIT.
		echo $PACKAGE $split
	fi
}


# scan_wok: one pass over every receipt feeding ALL the wok-wide outputs
# at once -- split.db + maint.db ($cache) + bdeps.txt + keep.list ($dbs).
# Replaces what used to be 4 separate full-wok scans (split, maint, bdeps
# and the toremove "second pass"), each sourcing ~6000 receipts. keep.list
# collects, per receipt, the tazpkg filenames a receipt still vouches for
# (every all_names() x {-$ARCH, -any}); packages.toremove is later derived
# from it with a single awk instead of thousands of in-loop `sed -i`.
# Needs $dbs and $arch set, so it runs from the main flow (not up here).

scan_wok() {
	local m names name
	: > "$cache/split.db"
	: > "$dbs/bdeps.txt"
	: > "$dbs/keep.list"
	: > "$dbs/maint.tmp"
	cd $WOK
	for pkg in *; do
		[ -s "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE SPLIT MAINTAINER BUILD_DEPENDS VERSION EXTRAVERSION
		. $WOK/$pkg/receipt

		names=$(all_names)
		echo -e "$PACKAGE\t$names" >> "$cache/split.db"

		# maintainer email between the angle brackets, no sed fork
		m=${MAINTAINER##*<}; m=${m%%>*}
		echo -e "$m\t$PACKAGE" >> "$dbs/maint.tmp"

		printf '%s\t%s\n' "$pkg" "$(echo $BUILD_DEPENDS)" >> "$dbs/bdeps.txt"

		for name in $names; do
			# also the bare (no-suffix) name: a 32-bit build in an
			# x86_64/arm chroot is tagged i486 by cook and named without
			# a suffix -- it belongs in this repo, don't let it be removed.
			printf '%s\n%s\n%s\n' \
				"$name-$VERSION$EXTRAVERSION$arch.tazpkg" \
				"$name-$VERSION$EXTRAVERSION-any.tazpkg" \
				"$name-$VERSION$EXTRAVERSION.tazpkg" >> "$dbs/keep.list"
		done
	done
	sort "$dbs/maint.tmp" > "$cache/maint.db"
	rm -f "$dbs/maint.tmp"
}


#
# Dedicated subcommands: exit early.
#

case "$1" in
	usage|help|--help|-h)  usage ;;
	--splitdb)  recreate_split_db; exit 0 ;;
	--maintdb)  recreate_maint_db; exit 0 ;;
esac


#
# Default mode: the wok-wide DBs (split.db, maint.db, bdeps.txt) and the
# toremove keep.list are all built in a single scan_wok pass below, once
# $dbs and $arch are set. Then the $PKGS lists are generated.
#
# Below: legacy `modules/pkgdb` body — packages dir lists generation.
#

# Find how much time was spent the last time (for web interface)
lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
[ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
while read cmd duration start; do
	[ $(($start + $duration)) -lt $(date +%s) ] &&
	echo "sed -i '/^$cmd $duration/d' $cooktime"
done < $cooktime | sh

rm $LOGS/pkgdb.log 2>/dev/null

case "$1" in
	--flavors|--rmpkg) ;;
	*)
		[ -n "$1" ] && PKGS="$1"
		if [ ! -d "$PKGS" ]; then
			{ newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
			exit 1
		fi ;;
esac

time=$(date +%s)
flavors="$SLITAZ/flavors"
live="$SLITAZ/live"

arch=''
case "$ARCH" in
	arm*|x86_64) arch="-$ARCH" ;;
esac

echo 'cook:pkgdb' > $command
_ 'Cook pkgdb: Creating all packages lists' | log
newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog

{ _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog

cd $PKGS

# Web interface database files should exist on the mirror1
>.folderlist; >.filelist
chmod 666 .folderlist .filelist

# Command `cook pkgdb` may be executed by cron. Creating a packages database
# takes some time, during which previously pending packages may continue to
# be created. This will result in an error due to the lack of a database. This in
# turn can lead to errors in the creating of many subsequent packages and will
# require manual intervention.
# Solution is an atomic update of the packages database: first we create new files
# (in the separate temp dir), and then instantly (well, almost) replace the old
# files by new ones.

dbs=$(mktemp -d)

# One scan over the whole wok: split.db + maint.db + bdeps.txt + keep.list.
action 'Refreshing wok databases...'
scan_wok
cd $PKGS		# scan_wok cd'd into $WOK; the find below needs $PKGS
status

_n 'Creating file "%s"' 'packages.list' | dblog
find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
echo " ($(filesize $dbs/packages.list))" | dblog

_n 'Creating file "%s"' 'packages.md5' | dblog
find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
echo " ($(filesize $dbs/packages.md5))" | dblog

md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs	# md5 and timestamp

_n 'Creating file "%s"' 'descriptions.txt' | dblog
for i in $(ls $WOK | sort); do
	[ -d "$WOK/$i/taz" ] || continue

	for j in $(ls $WOK/$i/taz | sort); do
		[ -e "$WOK/$i/taz/$j/description.txt" ] || continue

		pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
		echo "$pkgname"
		sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
		# if description.txt doesn't end with \n then add one
		[ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
		echo
	done >> $dbs/descriptions.txt
done
echo " ($(filesize $dbs/descriptions.txt))" | dblog

_n 'Creating file "%s"' 'bdeps.txt' | dblog		# built by scan_wok above
echo " ($(filesize $dbs/bdeps.txt))" | dblog


_ 'Creating lists from "%s"' "$WOK" | dblog
cd $WOK
rsumf=$(mktemp)
#touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
touch $dbs/packages.info $dbs/packages.equiv
for i in *; do
	[ -d "$WOK/$i/taz" ] || continue

	for j in $(ls $WOK/$i/taz | sort); do
		pack="$i/taz/$j"
		[ -f "$WOK/$pack/receipt" ] || continue
		unset_receipt
		unset PACKED_SIZE UNPACKED_SIZE RSUM PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC MAINTAINER LICENSE WEB_SITE DEPENDS TAGS PROVIDE
		. ./$pack/receipt

		# $arch is "" on i486 (no suffix, e.g. busybox-1.37.0.tazpkg) and
		# "-$ARCH" on x86_64/arm. -any.tazpkg covers noarch packages on
		# every arch.
		if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" -o \
		     -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg" -o \
		     -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then

			# packages.info combines TazPkg separate files
			# and will substitute them all
			SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
			DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts

#			#MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"

			# RSUM defined in new "small" receipts, but may be absent in old
			if [ -z "$RSUM" ]; then
				cp $pack/md5sum $rsumf
				md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
				[ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
				RSUM=$(md5sum $rsumf | awk '{print $1}')
			fi

			# i486 .tazpkg carry no arch suffix -> classic 10-field
			# record. x86_64/arm append field 11, the noarch flag:
			# "0" = -any.tazpkg, "1" = -$ARCH.tazpkg. tazpkg/get reads it
			# to rebuild the correct filename suffix (i486 needs none).
			# A 32-bit build in an x86_64/arm repo is tagged i486 by cook
			# -> bare name too, so it also gets the classic no-field-11 form.
			noarch=''
			if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg" ]; then
				noarch='\t0'
			elif [ -n "$arch" -a \
			       -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then
				noarch='\t1'
			fi
			printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%b\n' \
				"$PACKAGE" "$VERSION$EXTRAVERSION" "$CATEGORY" \
				"$SHORT_DESC" "$WEB_SITE" "$TAGS" "$SIZES" \
				"$DEPENDS" "$RSUM" "$PROVIDE" "$noarch" \
				>> $dbs/packages.info

			# packages.equiv is used by tazpkg install to check depends.
			for k in $PROVIDE; do
				DEST=''
				echo $k | fgrep -q : && DEST="${k#*:}:"
				if grep -qs ^${k%:*}= $dbs/packages.equiv; then
					sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
						$dbs/packages.equiv
				else
					echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
				fi
			done

			# files.list provides a list of all packages files.
			sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list

			# vouch for this built tazpkg (its taz/ version) so the
			# toremove awk below keeps it -- covers a version-skew case
			# scan_wok (source-receipt version) would miss.
			printf '%s\n%s\n%s\n' \
				"$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" \
				"$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg" \
				"$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" >> $dbs/keep.list
		else
			# The taz/ built-receipt copy strips WANTED and HOST_ARCH, so
			# consult the source receipt to judge arch relevance.
			unset WANTED HOST_ARCH
			eval "$(grep -E '^(WANTED|HOST_ARCH)=' "$WOK/$i/receipt" 2>/dev/null)"

			# A WANTED sub-package with no HOST_ARCH of its own is emitted
			# (or not) by its parent's per-arch build, not cooked directly --
			# e.g. the linux586* kernel splits only exist on i486. Missing
			# such a split here is the parent's per-arch decision, not a real
			# absence, so don't flag it (kills the linux586* noise flood on
			# x86_64/arm). Top-level packages and splits that explicitly claim
			# this arch via HOST_ARCH are still reported.
			if [ -n "$WANTED" -a -z "$HOST_ARCH" ]; then
				:
			# if receipt variable HOST_ARCH absent/empty or contains ARCH
			elif [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
				_ '  - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
			fi
		fi
	done
done


# Fallback: index the tazpkg files the loop above cannot see. A fresh
# wok clone has no taz/ dir for packages cooked before the clone (or
# built elsewhere and rsynced into $PKGS): they are shipped and listed
# in packages.list/md5, yet miss from packages.info and files.list --
# tazpkg clients then cannot install them. Their receipt, md5sum,
# description.txt and files.list all travel inside the .tazpkg itself,
# so extract these small members (not fs.cpio.lzma) and emit the very
# same records as the main loop.
cut -f1,2 $dbs/packages.info | sed 's|	|-|' > $rsumf.indexed
(cd $PKGS; ls *.tazpkg 2>/dev/null) | awk -v arch="$arch" '
	NR==FNR { idx[$0]=1; next }
	{
		f=$0; sub(/\.tazpkg$/, "", f)
		g=f; sub(/-any$/, "", g)
		if (arch != "") sub(arch "$", "", g)
		if (!(f in idx) && !(g in idx)) print
	}' $rsumf.indexed - > $rsumf.missing
if [ -s "$rsumf.missing" ]; then
	_ 'Indexing %s tazpkg without wok taz/ dir' "$(wc -l < $rsumf.missing)" | dblog
	tazx=$(mktemp -d)
	while read pkgfile; do
		rm -rf $tazx/*
		(cd $tazx; cpio -i receipt files.list md5sum description.txt \
			< $PKGS/$pkgfile 2>/dev/null)
		[ -s "$tazx/receipt" ] || continue
		# a corrupt archive receipt must not kill the whole pkgdb run
		# (e.g. a truncated receipt member): log it and skip.
		if ! sh -n $tazx/receipt 2>/dev/null; then
			_ '  - broken receipt: %s' "$pkgfile" | dblog
			continue
		fi
		unset_receipt
		unset PACKED_SIZE UNPACKED_SIZE RSUM PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC MAINTAINER LICENSE WEB_SITE DEPENDS TAGS PROVIDE
		. $tazx/receipt

		SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
		DEPENDS=$(echo $DEPENDS)
		if [ -z "$RSUM" ]; then
			cp $tazx/md5sum $rsumf
			md5sum $tazx/receipt | sed 's| [^ ]*/| |' >> $rsumf
			[ -e "$tazx/description.txt" ] && md5sum $tazx/description.txt | sed 's| [^ ]*/| |' >> $rsumf
			RSUM=$(md5sum $rsumf | awk '{print $1}')
		fi

		# same field-11 noarch flag as the main loop, from the filename
		noarch=''
		case "$pkgfile" in
			*-any.tazpkg) noarch='\t0' ;;
			*) [ -n "$arch" ] && case "$pkgfile" in
				*$arch.tazpkg) noarch='\t1' ;;
			esac ;;
		esac
		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%b\n' \
			"$PACKAGE" "$VERSION$EXTRAVERSION" "$CATEGORY" \
			"$SHORT_DESC" "$WEB_SITE" "$TAGS" "$SIZES" \
			"$DEPENDS" "$RSUM" "$PROVIDE" "$noarch" \
			>> $dbs/packages.info

		for k in $PROVIDE; do
			DEST=''
			echo $k | fgrep -q : && DEST="${k#*:}:"
			if grep -qs ^${k%:*}= $dbs/packages.equiv; then
				sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
					$dbs/packages.equiv
			else
				echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
			fi
		done

		[ -s "$tazx/files.list" ] &&
			sed "s|^|$PACKAGE: \0|" $tazx/files.list >> $dbs/files.list

		if [ -s "$tazx/description.txt" ]; then
			{
				echo "$PACKAGE"
				sed 's|^$| |' $tazx/description.txt
				[ -z "$(tail -c1 $tazx/description.txt)" ] || echo
				echo
			} >> $dbs/descriptions.txt
		fi
	done < $rsumf.missing
	rm -rf $tazx
	# keep files.list sorted by package name (pkgs.slitaz.org depends
	# on it): stable no-op on the already-sorted main-loop entries.
	LC_ALL=C sort -s -t: -k1,1 $dbs/files.list > $rsumf.sorted &&
		mv $rsumf.sorted $dbs/files.list
fi
rm -f $rsumf $rsumf.indexed $rsumf.missing


# packages.toremove = every tazpkg in packages.md5 that no receipt vouches
# for. keep.list was filled by scan_wok (all receipts' all_names x {-$ARCH,
# -any} -- this is the old "second pass" that kept packages whose taz/ was
# clean-wok'ed or pre-existing in $PKGS) plus the main loop (built taz/
# versions). One awk replaces the old ~7000 in-loop `sed -i` rewrites.
touch $dbs/keep.list
# A tazpkg is orphan only if neither its exact name NOR its arch-stripped name
# is vouched for. The strip rescues cross packages that cook tags by their
# CONTENT arch (farch), not the build arch: e.g. uclibc-x86_64-prebuilt-x86_64
# .tazpkg is cooked in the i486 chroot, but keep.list (built-arch suffixes only)
# holds uclibc-x86_64-prebuilt.tazpkg. Stripping the trailing -<arch> makes them
# match, so a valid receipt-backed build is not wrongly removed (and not counted
# unbuilt). A true orphan -- no receipt -- has no keep entry either way.
awk 'NR==FNR { keep[$0]=1; next }
	{
		if ($2 in keep) next
		g = $2; sub(/-[a-zA-Z0-9_]+\.tazpkg$/, ".tazpkg", g)
		if (g in keep) next
		print
	}' $dbs/keep.list $dbs/packages.md5 > $dbs/packages.toremove
rm -f $dbs/keep.list


# Display list size.
_ 'Done: %s (%s)' 'packages.info'  "$(filesize $dbs/packages.info)"  | dblog
_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog

cd $PKGS


# Check for unnecessary packages
if [ -s "$dbs/packages.toremove" ]; then
	newline | dblog
	case x$rmpkg in
		x) _ 'Found unnecessary packages (use `cook-pkgdb --rmpkg` to remove):' | dblog;;
		*) _ 'Removing unnecessary packages:' | dblog;;
	esac
	while read pkgsum pkgfile; do
		echo "  - $pkgfile" | dblog
		sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
		sed -i "/ $pkgfile/d" $dbs/packages.md5
		[ -n "$rmpkg" ] && rm $PKGS/$pkgfile	# remove packages only with --rmpkg
	done < $dbs/packages.toremove
	newline | dblog
fi
rm $dbs/packages.toremove


# packages.unbuilt = the symmetric of packages.toremove: every package an
# arch-marked receipt produces (split.db) that has NO cooked .tazpkg. Surfaces
# the "ghost" unbuilt -- a package whose deps or WANTED parent were broken so
# the cooker never retried it: no tazpkg, yet absent from the broken list and
# not caught by --rmpkg. Same source as the dashboard count (arch.$ARCH x
# split.db), so list and count stay consistent. Copied to $PKGS at the atomic
# update below, read by cooker.cgi / cook-wok.
ls $WOK/*/arch.$ARCH 2>/dev/null | sed 's|.*/\([^/]*\)/arch\.[^/]*$|\1|' | \
	awk 'NR==FNR { a[$1]=1; next } a[$1] { for (i=2; i<=NF; i++) print $i }' \
		- $cache/split.db | sort -u > $dbs/expected.names
# Race guard: the packages.list find() above ran early, before the ~6000-
# receipt main loop (seconds). A split cooked meanwhile by a concurrent
# in-chroot cook is on disk yet absent from that snapshot -> a false
# "unbuilt" until the next pass. Re-find now, right before the check, and
# match with the same version-strip awk.
find $PKGS -name '*.tazpkg' | sed 's|.*/||; s|.tazpkg$||' > $dbs/pkglist.now
awk 'NR==FNR { e[$0]=1; next }
	{ s=$0; while (match(s, /-[^-]*$/)) { s=substr(s, 1, RSTART-1); if (s in e) { c[s]=1; break } } }
	END { for (n in e) if (!(n in c)) print n }' \
	$dbs/expected.names $dbs/pkglist.now | sort > $dbs/packages.unbuilt
rm -f $dbs/expected.names $dbs/pkglist.now
[ -s "$dbs/packages.unbuilt" ] &&
	_ 'Unbuilt packages (receipt but no tazpkg): %s' "$(wc -l < $dbs/packages.unbuilt)" | dblog


# files.list.lzma
_n 'Creating file "%s"' 'files.list.lzma' | dblog
touch $dbs/files.list
# pkgs.slitaz.org strongly depends on list sorted by packages names
#lzma e $dbs/files.list $dbs/files.list.lzma
lzma -c $dbs/files.list > $dbs/files.list.lzma
echo " ($(filesize $dbs/files.list.lzma))" | dblog

# Pre-sorting filenames causes 10% smaller resulting lzma file
_n 'Creating file "%s"' 'files-list.lzma' | dblog
cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
#lzma e $dbs/files.list.sorted $dbs/files-list.lzma
lzma -c $dbs/files.list.sorted > $dbs/files-list.lzma
rm -f $dbs/files.list $dbs/files.list.sorted
echo " ($(filesize $dbs/files-list.lzma))" | dblog

md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5

# Bundle for `tazpkg recharge` clients — bakes upstream mirrors +
# extra.list into the local DB so clients fetch everything in one
# round-trip. Only useful on the official build host; gated by
# MAKE_BUNDLE in cook.conf. If mirror1 is unreachable we still skip
# gracefully rather than hang.
if [ "$MAKE_BUNDLE" = 'yes' ]; then
	_n 'Creating file "%s"' 'bundle.tar.lzma' | dblog

	# Fetch mirrors first with a SHORT timeout: it doubles as a
	# reachability probe. When mirror1 is down (chronic) the old
	# 2-files x 3-tries x 10s retry loops burned up to ~60s only to fall
	# back to the empty placeholders below anyway. If this single probe
	# fails, skip the rest of the fetches and go straight to placeholders.
	wget -q -T 5 -O $dbs/mirrors http://mirror1.slitaz.org/mirrors 2>/dev/null
	if [ -s "$dbs/mirrors" ]; then
		i=3
		while [ $i -gt 0 ] && [ ! -s "$dbs/extra.list" ]; do
			wget -q -T 10 -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
			echo -n '.' | dblog
			i=$((i - 1))
		done
	fi

	# Fall back to empty placeholders if mirror1 is unreachable: spk
	# (next-gen pkg manager) still wants bundle.tar.lzma and the local
	# DB files are what tazpkg/spk actually consume — mirrors and
	# extra.list are just upstream metadata that can be empty.
	[ -s "$dbs/mirrors" ]    || : > $dbs/mirrors
	[ -s "$dbs/extra.list" ] || : > $dbs/extra.list

	(
		cd $dbs
		busybox tar -chf bundle.tar \
			mirrors extra.list files-list.md5 packages.info \
			descriptions.txt packages.md5 packages.list packages.equiv
	)
	lzma -c $dbs/bundle.tar > $dbs/bundle.tar.lzma
	rm $dbs/bundle.tar $dbs/mirrors
	echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
fi

# Display some info.
separator | dblog
nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
time=$(($(date +%s) - $time))
# L10n: 's' is for seconds (cooking time)
{ _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog

# "Atomic" update now
ls -l $dbs
cp -f $dbs/* $PKGS
rm -r $dbs


# Create all flavors files at once. Do we really need code to monitor
# flavors changes? Let's just build them with packages lists before
# syncing the mirror.
[ "$1" != '--flavors' ] && rm $command && exit 1

if [ ! -d "$flavors" ]; then
	{ _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
	rm $command
	exit 1
fi

[ ! -d "$live" ] && mkdir -p $live
_ 'Creating flavors files in "%s"' "$live" | dblog
_ 'Cook pkgdb: Creating all flavors' | log
separator | dblog

_ 'Recharging lists to use latest packages...' | dblog
tazpkg recharge >/dev/null 2>/dev/null

# We need a custom tazlito config to set working dir to /home/slitaz.
if [ ! -f "$live/tazlito.conf" ]; then
	_ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
	cp /etc/tazlito/tazlito.conf $live
	sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
		$live/tazlito.conf
fi

# Update Hg flavors repo and pack.
if [ -d "$flavors/.hg" ]; then
	cd $flavors; hg pull -u
fi

cd $live
_ 'Starting to generate flavors...' | dblog
rm -f flavors.list *.flavor
for i in $flavors/*; do
	fl=$(basename $i)
	_ 'Packing flavor "%s"' "$fl" | dblog
	tazlito pack-flavor $fl >/dev/null || exit 1
	tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
done
cp -f $live/*.flavor $live/flavors.list $PKGS
separator | dblog
{ _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
separator | dblog
_ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog

rm $command


exit 0
