#!/bin/ksh

# This script create a (partial) debmirror using
# deb packages files present on the system.
# Unlike "apt-move mirror", this script doesn't download
# any package from debian servers. It only downloads packages
# index files from http repositories servers listed in
# /etc/apt/sources.list.
# 
# It looks at different directories to find new packages
#  /var/cache/apt/archives/
#  /var/cache/apt-proxy/debian/
# So re-run this script after installing new packages
# or after installing new machines by FAI through apt-proxy.
#
# You can copy all *.deb from debian cdroms into /var/cache/apt/archives/
# to have an initial package file set.
#
# It is recommended to host you debmirror on the same
# filesystem than /var/cache/apt/archives/ as the script
# tries to create hard links to save disk space. If a
# hard link doesn't work, packages are copied.
#
# This script excludes some packages that I don't care about
# (mainly : games, asian specific tools, ...). You can add
# or remove some other packages directly in the script to sweet
# your needs.

# ---------------------------------------------------------------
# SETTINGS :

# If you need to go through a web proxy, define one
# of these variable :
# export http_proxy=http://proxy:port/ # simple proxy
# export http_proxy=http://user:passwd@proxy:port/ # proxy with authentication

# The location of you local debmirror
DEBMIRROR_DIR=/debmirror

# ---------------------------------------------------------------

< /dev/null

# This fonction looks at file /etc/apt/sources.list
# to determine the distributions and the branches to mirror
# The output of this function is a set of lines of the form :
#  server-http dist branch
branch_list()
{
	cat /etc/apt/sources.list | egrep '^[ 	]*deb *http' | sed -e 's/#.*$//' | while read line
	do
		mkdir -p $DEBMIRROR_DIR/.data
		line=${line#deb }
		echo "$line" | read http dist branches
		http=${http%/}
		for branch in $branches
		{
			echo $http $dist $branch
		}
	done
}

download_pkg_index()
{
	http=$1
	dist=$2
	branch=$3
	datafile=$4

	mkdir -p $DEBMIRROR_DIR/.data

	(
		cd $DEBMIRROR_DIR/.data
		[ -f $datafile ] && mv $datafile Packages.gz
		wget -qN $http/dists/$dist/$branch/binary-i386/Packages.gz && echo ok || echo failed
		mv Packages.gz $datafile
	)

}

test_package_present()
{
	paragraph=""
	empty=y
	while IFS='
' read -r line
	do
		case "$line" in
		Package*)
			pkg="${line##Package: }"
			;;
		Filename:*)
			filename="${line##Filename: }"
			;;
		esac
		paragraph="$paragraph
$line"
		if [ "$line" == "" -a "$filename" != "" ]
		then
			if [ -f $DEBMIRROR_DIR/$filename ]
			then
				printf '%s' "$paragraph"
				empty=n
			fi
			filename=""
			paragraph=""
		fi
	done
	# Ensure that we have a newline at the file
	if [ $empty == n ] ; then
		echo
	fi
}

foreach_package()
{
	action=$1
	oldpn=''
	shift

	while read line
	do
		case "$line" in
		Package*)
			pkg="${line##Package: }"
			;;
		Filename:*)
			filename="${line##Filename: }"
			;;
		Section:*)
			section="${line##Section: }"
			;;
		Task:*)
			task="${line##Task: }"
			;;
		Priority:*)
			priority="${line##Priority: }"
			;;
		"")
			$action $pkg $filename "$section" "$task" "$priority" $@
			newpn=$pkg
			typeset -L1 newpn
			if [[ "$newpn" > "$oldpn" ]] ; then
				printf .
				oldpn=$newpn
			fi
			filename=""
			section=""
			task=""
			priority=""
			;;
		esac
	done
	echo
}

pkg_to_pool()
{
	exclude=n
	package=$1
	server="$6"

	case $package in
	zope*|kernel-patch*|phpgroupware*|atari*|timidity*|gmt*|ircii*|*-ia64*|*-k7|*-hppa|*-sparc|kernel-*-2.2*|kernel-*-2.5*)
		exclude="is not mirrored"
		;;
	xfont-base)
		# keep this package (Special hack for package xfonts-base
		# which belongs to task Japanese... I don't know why)
		;;
	*)
		case $task in
		*french*)
			# keep this one
			;;
		russian|polish|junior|japanese|chinese-*)
			exclude="Task $task"
			;;
		esac

		case $section in
		*games|*hamradio|*electronics)
			exclude="Section $section"
			;;
		esac

	esac

	if [ "$exclude" != n ]
	then
		# We don't mirror games, russian, ...
		if [ -f "$DEBMIRROR_DIR/$filename" ]
		then
			print excluding $package : $exclude
			rm -f $DEBMIRROR_DIR/$filename
		fi
	else
		if [ ! -f $DEBMIRROR_DIR/$filename ]
		then
			basename=$(basename $filename)
			dirname=$(dirname $filename)
			source=""

			# the list of possible source for local packages
			# add some more here
			if [ -f /var/cache/apt/archives/$basename ]
			then
				source=/var/cache/apt/archives/$basename
			elif [ -f /local00/debmirror/$filename ]
			then
				source=/local00/debmirror/$filename
			elif [ -f /var/cache/apt-proxy/debian/$filename ]
			then
				source=/var/cache/apt-proxy/debian/$filename
			elif [ -f /var/cache/apt-proxy/security/$filename ]
			then
				source=/var/cache/apt-proxy/security/$filename
			fi

			if [ "$source" == "" ]
			then
				echo pkg="$package" file=$server/$filename >> $DEBMIRROR_DIR/.data/missing
			else
				mkdir -p $DEBMIRROR_DIR/$dirname
				# trying to make a hard link ; if this fails, try to copy the file
				#echo "Adding $package"
				ln $source $DEBMIRROR_DIR/$dirname || cp -p $source $DEBMIRROR_DIR/$dirname
			fi
		fi
		if [ -f $DEBMIRROR_DIR/$filename ]
		then
			echo pkg="$package" file=$filename >> $DEBMIRROR_DIR/.data/present
		fi
	fi
}

create_top_release()
{
	branch=$2
	cat << EOF
Origin: APT-Move
Label: APT-Move
Suite: stable
Codename: unknown
Date: Thu, 21 Aug 2003 16:45:01 UTC
Architectures: i386
Components: main contrib non-free
Description: unknown
EOF
	file_to_sum=$(find $1 \( -name Packages\* -o -name Release \) -print )
	echo 'MD5Sum:'
	for l in $file_to_sum
	{
		size=$(stat -c %s $l)
		f=${l##$1}
		printf ' %32s%.s %16d %s\n' $(md5sum $l) $size $f
	}
	echo 'SHA1:'
	for l in $file_to_sum
	{
		size=$(stat -c %s $l)
		f=${l##$1}
		printf ' %40s%.s %16d %s\n' $(sha1sum $l) $size $f
	}
}

create_branch_release()
{
cat << EOF
Archive: $1
Component: $2
Origin: APT-Move
Label: APT-Move
Architecture: i386
EOF
}

create_dist_aliases()
{
	mkdir -p $DEBMIRROR_DIR/dists/stable
	ln -s stable $DEBMIRROR_DIR/dists/woody 2>/dev/null
	mkdir -p $DEBMIRROR_DIR/dists/oldstable
	ln -s oldstable $DEBMIRROR_DIR/dists/potato 2>/dev/null
	mkdir -p $DEBMIRROR_DIR/dists/stable-proposed-updates
	ln -s stable-proposed-updates $DEBMIRROR_DIR/dists/woody-proposed-updates 2>/dev/null
}

# ---------------------------------------------------------------

# Verify that the debian mirror root directory exists

if [ ! -d $DEBMIRROR_DIR ] ; then
	echo "Debian mirror directory does not exist"
	exit 1
fi

# Some clean ups
rm -rf $DEBMIRROR_DIR/.data/missing $DEBMIRROR_DIR/.data/present
find $DEBMIRROR_DIR/dists \( -type l -o \( -type f \( -name Package\* -o -name Release \) \) \) -exec rm {} \;

create_dist_aliases

branch_list | while read server dist branch; do
	if [ "$olddist" != "$dist" ]
	then
		echo " - Processing distribution : $dist"
		dist_list="$dist_list $dist"
	fi
	echo "   - branch : $branch"
	datafile=$DEBMIRROR_DIR/.data/$(echo ${server#http://}/${dist}/${branch}/Packages.gz | tr / _ )
	printf "     - Downloading package list : "
	download_pkg_index $server $dist $branch $datafile
	printf "     - Updating package pool"
	zcat $datafile| foreach_package pkg_to_pool ${server}
	output_dir=$DEBMIRROR_DIR/dists/${dist}/${branch}/binary-i386
	mkdir -p $output_dir
	echo "     - Creating Packages and Release files"
	zcat $datafile| test_package_present >> $output_dir/Packages
	gzip -9 -c $output_dir/Packages > $output_dir/Packages.gz
	create_branch_release ${dist} ${branch} > $output_dir/Release

	olddist=$dist
done

for dist in $dist_list
{
	create_top_release $DEBMIRROR_DIR/dists/${dist}/ > $DEBMIRROR_DIR/dists/${dist}/Release
}