head	1.23;
access;
symbols
	pkgsrc-2026Q1:1.23.0.2
	pkgsrc-2026Q1-base:1.23
	pkgsrc-2025Q4:1.21.0.34
	pkgsrc-2025Q4-base:1.21
	pkgsrc-2025Q3:1.21.0.32
	pkgsrc-2025Q3-base:1.21
	pkgsrc-2025Q2:1.21.0.30
	pkgsrc-2025Q2-base:1.21
	pkgsrc-2025Q1:1.21.0.28
	pkgsrc-2025Q1-base:1.21
	pkgsrc-2024Q4:1.21.0.26
	pkgsrc-2024Q4-base:1.21
	pkgsrc-2024Q3:1.21.0.24
	pkgsrc-2024Q3-base:1.21
	pkgsrc-2024Q2:1.21.0.22
	pkgsrc-2024Q2-base:1.21
	pkgsrc-2024Q1:1.21.0.20
	pkgsrc-2024Q1-base:1.21
	pkgsrc-2023Q4:1.21.0.18
	pkgsrc-2023Q4-base:1.21
	pkgsrc-2023Q3:1.21.0.16
	pkgsrc-2023Q3-base:1.21
	pkgsrc-2023Q2:1.21.0.14
	pkgsrc-2023Q2-base:1.21
	pkgsrc-2023Q1:1.21.0.12
	pkgsrc-2023Q1-base:1.21
	pkgsrc-2022Q4:1.21.0.10
	pkgsrc-2022Q4-base:1.21
	pkgsrc-2022Q3:1.21.0.8
	pkgsrc-2022Q3-base:1.21
	pkgsrc-2022Q2:1.21.0.6
	pkgsrc-2022Q2-base:1.21
	pkgsrc-2022Q1:1.21.0.4
	pkgsrc-2022Q1-base:1.21
	pkgsrc-2021Q4:1.21.0.2
	pkgsrc-2021Q4-base:1.21
	pkgsrc-2021Q3:1.20.0.10
	pkgsrc-2021Q3-base:1.20
	pkgsrc-2021Q2:1.20.0.8
	pkgsrc-2021Q2-base:1.20
	pkgsrc-2021Q1:1.20.0.6
	pkgsrc-2021Q1-base:1.20
	pkgsrc-2020Q4:1.20.0.4
	pkgsrc-2020Q4-base:1.20
	pkgsrc-2020Q3:1.20.0.2
	pkgsrc-2020Q3-base:1.20
	pkgsrc-2020Q2:1.19.0.16
	pkgsrc-2020Q2-base:1.19
	pkgsrc-2020Q1:1.19.0.12
	pkgsrc-2020Q1-base:1.19
	pkgsrc-2019Q4:1.19.0.14
	pkgsrc-2019Q4-base:1.19
	pkgsrc-2019Q3:1.19.0.10
	pkgsrc-2019Q3-base:1.19
	pkgsrc-2019Q2:1.19.0.8
	pkgsrc-2019Q2-base:1.19
	pkgsrc-2019Q1:1.19.0.6
	pkgsrc-2019Q1-base:1.19
	pkgsrc-2018Q4:1.19.0.4
	pkgsrc-2018Q4-base:1.19
	pkgsrc-2018Q3:1.19.0.2
	pkgsrc-2018Q3-base:1.19
	pkgsrc-2018Q2:1.18.0.26
	pkgsrc-2018Q2-base:1.18
	pkgsrc-2018Q1:1.18.0.24
	pkgsrc-2018Q1-base:1.18
	pkgsrc-2017Q4:1.18.0.22
	pkgsrc-2017Q4-base:1.18
	pkgsrc-2017Q3:1.18.0.20
	pkgsrc-2017Q3-base:1.18
	pkgsrc-2017Q2:1.18.0.16
	pkgsrc-2017Q2-base:1.18
	pkgsrc-2017Q1:1.18.0.14
	pkgsrc-2017Q1-base:1.18
	pkgsrc-2016Q4:1.18.0.12
	pkgsrc-2016Q4-base:1.18
	pkgsrc-2016Q3:1.18.0.10
	pkgsrc-2016Q3-base:1.18
	pkgsrc-2016Q2:1.18.0.8
	pkgsrc-2016Q2-base:1.18
	pkgsrc-2016Q1:1.18.0.6
	pkgsrc-2016Q1-base:1.18
	pkgsrc-2015Q4:1.18.0.4
	pkgsrc-2015Q4-base:1.18
	pkgsrc-2015Q3:1.18.0.2
	pkgsrc-2015Q3-base:1.18
	pkgsrc-2015Q2:1.15.0.20
	pkgsrc-2015Q2-base:1.15
	pkgsrc-2015Q1:1.15.0.18
	pkgsrc-2015Q1-base:1.15
	pkgsrc-2014Q4:1.15.0.16
	pkgsrc-2014Q4-base:1.15
	pkgsrc-2014Q3:1.15.0.14
	pkgsrc-2014Q3-base:1.15
	pkgsrc-2014Q2:1.15.0.12
	pkgsrc-2014Q2-base:1.15
	pkgsrc-2014Q1:1.15.0.10
	pkgsrc-2014Q1-base:1.15
	pkgsrc-2013Q4:1.15.0.8
	pkgsrc-2013Q4-base:1.15
	pkgsrc-2013Q3:1.15.0.6
	pkgsrc-2013Q3-base:1.15
	pkgsrc-2013Q2:1.15.0.4
	pkgsrc-2013Q2-base:1.15
	pkgsrc-2013Q1:1.15.0.2
	pkgsrc-2013Q1-base:1.15
	pkgsrc-2012Q4:1.14.0.28
	pkgsrc-2012Q4-base:1.14
	pkgsrc-2012Q3:1.14.0.26
	pkgsrc-2012Q3-base:1.14
	pkgsrc-2012Q2:1.14.0.24
	pkgsrc-2012Q2-base:1.14
	pkgsrc-2012Q1:1.14.0.22
	pkgsrc-2012Q1-base:1.14
	pkgsrc-2011Q4:1.14.0.20
	pkgsrc-2011Q4-base:1.14
	pkgsrc-2011Q3:1.14.0.18
	pkgsrc-2011Q3-base:1.14
	pkgsrc-2011Q2:1.14.0.16
	pkgsrc-2011Q2-base:1.14
	pkgsrc-2011Q1:1.14.0.14
	pkgsrc-2011Q1-base:1.14
	pkgsrc-2010Q4:1.14.0.12
	pkgsrc-2010Q4-base:1.14
	pkgsrc-2010Q3:1.14.0.10
	pkgsrc-2010Q3-base:1.14
	pkgsrc-2010Q2:1.14.0.8
	pkgsrc-2010Q2-base:1.14
	pkgsrc-2010Q1:1.14.0.6
	pkgsrc-2010Q1-base:1.14
	pkgsrc-2009Q4:1.14.0.4
	pkgsrc-2009Q4-base:1.14
	pkgsrc-2009Q3:1.14.0.2
	pkgsrc-2009Q3-base:1.14
	pkgsrc-2009Q2:1.12.0.6
	pkgsrc-2009Q2-base:1.12
	pkgsrc-2009Q1:1.12.0.4
	pkgsrc-2009Q1-base:1.12
	pkgsrc-2008Q4:1.12.0.2
	pkgsrc-2008Q4-base:1.12
	pkgsrc-2008Q3:1.10.0.2
	pkgsrc-2008Q3-base:1.10
	cube-native-xorg:1.9.0.12
	cube-native-xorg-base:1.9
	pkgsrc-2008Q2:1.9.0.10
	pkgsrc-2008Q2-base:1.9
	cwrapper:1.9.0.8
	pkgsrc-2008Q1:1.9.0.6
	pkgsrc-2008Q1-base:1.9
	pkgsrc-2007Q4:1.9.0.4
	pkgsrc-2007Q4-base:1.9
	pkgsrc-2007Q3:1.9.0.2
	pkgsrc-2007Q3-base:1.9
	pkgsrc-2007Q2:1.8.0.6
	pkgsrc-2007Q2-base:1.8
	pkgsrc-2007Q1:1.8.0.4
	pkgsrc-2007Q1-base:1.8
	jlam-pkgviews:1.8.0.2
	jlam-pkgviews-base:1.8
	pkgsrc-2006Q4:1.6.0.4
	pkgsrc-2006Q4-base:1.6
	pkgsrc-2006Q3:1.6.0.2
	pkgsrc-2006Q3-base:1.6;
locks; strict;
comment	@# @;


1.23
date	2026.02.15.09.46.33;	author wiz;	state Exp;
branches;
next	1.22;
commitid	PA6JiOXzggNfksuG;

1.22
date	2026.02.07.15.15.02;	author wiz;	state Exp;
branches;
next	1.21;
commitid	h46jVA2rhrIopstG;

1.21
date	2021.12.16.14.10.46;	author schmonz;	state Exp;
branches;
next	1.20;
commitid	UORVrPS32zJBISkD;

1.20
date	2020.08.27.11.45.45;	author jperkin;	state Exp;
branches;
next	1.19;
commitid	82sM4xDhiUVa7HlC;

1.19
date	2018.08.22.20.48.37;	author maya;	state Exp;
branches;
next	1.18;
commitid	FMEpDDTKwDQBR9PA;

1.18
date	2015.09.19.11.45.56;	author dsainty;	state Exp;
branches;
next	1.17;
commitid	CAIquDVqjcnYbRBy;

1.17
date	2015.09.08.12.37.56;	author joerg;	state Exp;
branches;
next	1.16;
commitid	llJvmeK5NpAjQrAy;

1.16
date	2015.09.08.11.13.12;	author joerg;	state Exp;
branches;
next	1.15;
commitid	0Sy43Eq9ArX3nrAy;

1.15
date	2013.02.16.07.32.28;	author obache;	state Exp;
branches;
next	1.14;

1.14
date	2009.08.02.02.38.52;	author joerg;	state Exp;
branches;
next	1.13;

1.13
date	2009.07.20.09.09.02;	author tnn;	state Exp;
branches;
next	1.12;

1.12
date	2008.12.15.09.01.30;	author taca;	state Exp;
branches;
next	1.11;

1.11
date	2008.12.14.14.09.32;	author rillig;	state Exp;
branches;
next	1.10;

1.10
date	2008.09.15.21.07.46;	author rillig;	state Exp;
branches;
next	1.9;

1.9
date	2007.08.04.15.00.29;	author obache;	state Exp;
branches;
next	1.8;

1.8
date	2007.01.18.10.51.48;	author rillig;	state Exp;
branches;
next	1.7;

1.7
date	2007.01.10.14.31.35;	author rillig;	state Exp;
branches;
next	1.6;

1.6
date	2006.08.03.14.02.58;	author jlam;	state Exp;
branches;
next	1.5;

1.5
date	2006.07.20.13.31.08;	author joerg;	state Exp;
branches;
next	1.4;

1.4
date	2006.07.19.15.13.40;	author jlam;	state Exp;
branches;
next	1.3;

1.3
date	2006.07.19.14.16.29;	author jlam;	state Exp;
branches;
next	1.2;

1.2
date	2006.07.18.22.41.06;	author jlam;	state Exp;
branches;
next	1.1;

1.1
date	2006.07.17.15.34.22;	author jlam;	state Exp;
branches;
next	;


desc
@@


1.23
log
@mk: fetch to temporary file

Download to a temporary file and move the file to the final location
when the download is finished - so the file is there or not, but not a
partially downloaded copy.

The script already supported resuming downloads - the support is still
there but basically will never happen again, because it only looks at
a partial file of the correct name (which can still happen if the
download finishes but provides an incomplete file, and the file cannot
be verified by during download e.g. because the distinfo file is not
available - a theoretically supported use case)

As proposed on tech-pkg.
@
text
@#!/bin/sh
#
# $NetBSD: fetch,v 1.22 2026/02/07 15:15:02 wiz Exp $
#
# Copyright (c) 2006, 2015 The NetBSD Foundation, Inc.
# All rights reserved.
#
# This code is derived from software contributed to The NetBSD Foundation
# by Johnny C. Lam.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

######################################################################
#
# NAME
#	fetch -- fetch files via URLs
#
# SYNOPSIS
#	fetch [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]
#
# DESCRIPTION
#	fetch will attempt to fetch the file from the list of specified
#	sites in the order given.  The complete URL to the file on each
#	site should be the concatenation of the specified site and file.
#	If the file cannot be fetched successfully, then we try the next
#	listed site.
#
#	The file will be downloaded under a temporary name and
#	renamed to the proper name when the download is done.
#
#	If the file already exists on the disk and is verified, then
#	no fetch action is taken.
#
# OPTIONS
#	-c	Verify the checksum for the file.  If the checksum
#		does not match, then the fetch is determined to be
#		not successful.
#
#	-d dir	Fetch the files into the specified directory.
#
#	-f distinfo
#		The path to the distinfo file containing the checksums
#		for the file.  The file format should match what is
#		needed by the pkgsrc/mk/checksum/checksum script.
#
#	-p hook
#		After a successful fetch, run hook.  The first argument is
#		the relative path of the distfile and the second argument
#		the full URL the file was obtained from.
#
#	-r	Resume a previous fetch for the file.  In this case,
#		the file is first saved to a ".pkgsrc.resume" file,
#		and is later renamed to the final file name if the
#		complete file has been successfully fetched.
#
#	-v	Show the actual command line used to fetch the file
#		from each site.
#
# ENVIRONMENT
#	PKGSRCDIR	This is a hint to help locate the default
#			checksum script.
#
#	CHECKSUM	This is the path to the checksum script used
#			when "-c" is specified.
#
#	FETCH_CMD	This is the actual command used for transferring
#			files from the various sites.
#
#	The following are lists of options to pass to ${FETCH_CMD}:
#
#	FETCH_BEFORE_ARGS
#			These options appear before all other options.
#
#	FETCH_AFTER_ARGS
#			These options appear after all other options.
#
#	FETCH_RESUME_ARGS
#			These options appear just after FETCH_BEFORE_ARGS
#			options and cause ${FETCH_CMD} to resume a
#			previous file transfer.
#
#	FETCH_OUTPUT_ARGS
#			These options specify the name of the local file
#			that will hold the contents of the fetched file.
#
######################################################################

: ${PKGSRCDIR:=/usr/pkgsrc}
: ${CHECKSUM:=false}
: ${CP:=cp}
: ${ECHO:=echo}
: ${FETCH_CMD:=ftp}
: ${MKDIR:=mkdir}
: ${MV:=mv}
: ${TEST:=test}
: ${TOUCH:=touch}
: ${WC:=wc}

self="${0##*/}"

usage() {
	${ECHO} 1>&2 "usage: $self [-crv] [-d dir] [-f distinfo] [-p hook] file [site ...]"
}

# Process optional arguments
checksum=
distinfo=
fetchdir=.			# A relative directory or "."
post_fetch=
resume=
verbose=
while ${TEST} $# -gt 0; do
	case "$1" in
	-c)	checksum=yes; shift ;;
	-d)	fetchdir="$2"; shift 2 ;;
	-f)	distinfo="$2"; shift 2 ;;
	-p)	post_fetch="$2"; shift 2 ;;
	-r)	resume=yes; shift ;;
	-v)	verbose=yes; shift ;;
	--)	shift; break ;;
	-*)	${ECHO} 1>&2 "$self: unknown option -- ${1#-}"
		usage
		exit 1
		;;
	*)	break ;;
	esac
done
if ${TEST} -n "$checksum" -a -z "$distinfo"; then
	${ECHO} 1>&2 "$self: \`\`-c'' requires \`\`-f distinfo''."
	exit 1
fi
if ${TEST} -n "$resume"; then
	if ${TEST} -z "$distinfo"; then
		${ECHO} 1>&2 "$self: \`\`-r'' requires \`\`-f distinfo''."
		resume=
	elif ${TEST} "x${FETCH_RESUME_ARGS}" = "x"; then
		${ECHO} 1>&2 "$self: \`\`-r'' requires FETCH_RESUME_ARGS to be non-empty."
		resume=
	fi
	${TEST} -n "$resume" ||
		${ECHO} 1>&2 "$self: Falling back to non-resume fetch."
fi

# Process required arguments
if ${TEST} $# -lt 1; then
	usage
	exit 1
fi
file="$1"; shift
path="$fetchdir/$file"

if ${TEST} -n "$distinfo" && ${TEST} ! -f "$distinfo"; then
	${ECHO} 1>&2 "$self: distinfo file missing: $distinfo"
	exit 1
fi

# Compute the expected size of the fetched file.
distsize=
distunits=
if ${TEST} -n "$distinfo"; then
	while read d_type d_file d_equals d_size d_units; do
		case "$d_type" in
		Size)	;;	# only handle "Size" lines
		*)	continue ;;
		esac
		case "$fetchdir" in
		".")	${TEST} "$d_file" = "($file)" || continue ;;
		*)	${TEST} "$d_file" = "($path)" || continue ;;
		esac
		distsize="$d_size"; distunits="$d_units"
		break
	done < $distinfo
fi

# verify_file [-v] $file $suffix
#	If we can checksum the file, then see if it matches the listed
#	checksums in the distinfo file.  If we can check the size, then
#	check that instead.  We strip off ".pkgsrc.resume" from the
#	filename so that we can verify the checksum for the temporary
#	fetch file as well.
#
verify_file() {
	_if_verbose=:; if [ "x$1" = "x-v" ]; then shift; _if_verbose=; fi
	_file="${1#./}"
	_suffix="$2"
	${TEST} -f $_file || {
		$_if_verbose ${ECHO} 1>&2 "$self: File $_file does not exist."
		return 1
	}
	if ${TEST} -n "$checksum"; then
		${CHECKSUM} -s "${_suffix}" $distinfo ${_file} || {
			$_if_verbose ${ECHO} 1>&2 "$self: Checksum of the file $_file doesn't match."
			return 1
		}
		return 0
	elif ${TEST} -n "$distsize"; then
		_size=`${WC} -c < $_file`
		${TEST} "$_size" -eq "$distsize" || {
			$_if_verbose ${ECHO} 1>&2 "$self: Size of the file $_file doesn't match."
			return 1
		}
		return 0
	fi
	return 0;
}

# If the file already exists and it verifies, then we don't need to fetch
# it again.
#
if verify_file $path ""; then
	exit 0
fi

${TEST} -d $fetchdir || ${MKDIR} -p $fetchdir 2>/dev/null

${TEST} -w $fetchdir || ${ECHO} 1>&2 "$self: WARNING: DISTDIR `cd $fetchdir && pwd` looks non-writable."

# Set the name of the output file.  In the "resume" case, we initialize
# the fetch loop by providing the existing, possibly incomplete, file.
#
tmp_suffix=".$$.part"
outputfile="$file$tmp_suffix"
outputpath="$fetchdir/$outputfile"
if ${TEST} -f $outputpath; then
    ${ECHO} "Temporary download file ${outputpath} already existed, deleting"
    ${RM} -f $outputpath
fi
if ${TEST} -n "$resume"; then
	if ${TEST} -f $path; then
		${CP} -f $path $outputpath
	else
		${TOUCH} $outputpath
	fi
	size=`${WC} -c < $outputpath`
	${ECHO} "=> Downloaded size (before resume): $size bytes"
fi
${TEST} -z "$distsize" || ${ECHO} "=> Total size: $distsize $distunits"

# Iterate over each site and try to fetch the file.  We verify the fetched
# file to see if we need to try fetching from the next site.
#
while ${TEST} $# -gt 0; do
	site="$1"; shift
	case "$site" in
	-*)
		url=${site#-}
		;;
	*)
		url=$site$file
		;;
	esac

	( cd $fetchdir
	  if ${TEST} -n "$resume"; then
		fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url"
	  else
		fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url ${FETCH_AFTER_ARGS}"
	  fi
	  ${TEST} -z "$verbose" || ${ECHO} "$fetch_cmd"
	  $fetch_cmd )
	if ${TEST} $? -ne 0; then
		${ECHO} 1>&2 "$self: Unable to fetch expected file $file"
		continue
	fi
	if verify_file -v $outputpath $tmp_suffix; then
		${MV} -f $outputpath $path
		break
	fi
	if ${TEST} -n "$resume"; then
		if ${TEST} -f $path; then
			${CP} -f $path $outputpath
		else
			${TOUCH} $outputpath
		fi
	fi
done
if ${TEST} -f $path && ${TEST} -n "$post_fetch"; then
	"$post_fetch" $path "$url"
fi
if ${TEST} -f $path; then
	exit 0
else
	exit 1
fi
@


1.22
log
@mk: fetch: fix typo in comment, simplify usage
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.21 2021/12/16 14:10:46 schmonz Exp $
d48 3
d195 1
a195 1
# verify_file [-v] $file
d205 1
d211 1
a211 1
		${CHECKSUM} -s ".pkgsrc.resume" $distinfo ${_file} || {
d230 1
a230 1
if verify_file $path; then
d239 1
a239 2
# the fetch loop by ensuring that the temporary output file already
# exists.
d241 2
a242 1
outputfile="$file"
d244 4
d249 4
a252 17
	outputfile="${file}.pkgsrc.resume"
	outputpath="$fetchdir/$outputfile"
	if ${TEST} ! -f $outputpath; then
		if ${TEST} -f $path; then
			${CP} -f $path $outputpath
		else
			${RM} -f $outputpath
			${TOUCH} $outputpath
		fi
	fi
	#
	# If the temporary file verifies, then we don't need to resume
	# fetching it.
	#
	if verify_file $outputpath; then
		${MV} -f $outputpath $path
		exit 0
d255 1
a255 1
	${ECHO} "=> Downloaded size: $size bytes"
d285 2
a286 2
	if verify_file -v $outputpath; then
		${TEST} -z "$resume" || ${MV} -f $outputpath $path
@


1.21
log
@The non-writability check for DISTDIR is intended to prevent wasting
resources when fetchers wouldn't be able to save what they download. On
my build farm with pkgsrc (and pkgsrc/distfiles) mounted over NFS,
however, the check gives false positives for NetBSD 9.2 and -current.
Downgrade it to a warning so that these fetches can succeed, while
leaving a breadcrumb in case someone encounters a true positive.

It would of course be interesting to sort out why, in my environment, a
wide variety of other OSes get 1 for "${TEST} ! -w $fetchdir" while
NetBSD gets 0. In the meantime, joerg@@ suggested this workaround and
gdt@@ agrees it's reasonable to try.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.20 2020/08/27 11:45:45 jperkin Exp $
d71 1
a71 1
#		complete file has been sucessfully fetched.
d119 1
a119 1
	${ECHO} 1>&2 "usage: $self [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]"
@


1.20
log
@mk: Rewrite the checksum script in awk.

The previous shell script version's runtime was quadratic against the
number of distfiles to verify.  Historically this has not been an issue,
with usually only a handful of files per package.  However, with the
introduction of Go modules the number of distfiles used by a single
package can be very high.

For example, in an upcoming update of www/grafana to version 7.1.5, the
number of GO_MODULE_FILES is 821.  Running 'bmake checksum' takes:

  real    18m20.743s
  user    17m27.975s
  sys     0m49.239s

With the awk code, this is reduced to a far more sensible:

  real    0m4.330s
  user    0m3.241s
  sys     0m0.875s

The script has been written to emulate the previous version precisely,
preserving the same output and error messages and supporting all of its
behaviour, with the one exception that previous exit values of 128 have
been changed to 3, in order to avoid any potential signed 8-bit issues.

The one change in the pkgsrc infrastructure is that the mk/fetch/fetch
script no longer sets a working default value for ${CHECKSUM}.  This is
not a problem in a pkgsrc environment as all of the required variables
are set correctly, but if there happen to be any users who are using
this script in a standalone environment, they will need to set it
accordingly.  This was probably required in many situations previously
anyway, as none of the script's environment variables were set, and
trying to support this would be fragile at best.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.19 2018/08/22 20:48:37 maya Exp $
d231 2
a232 4
if ${TEST} ! -w $fetchdir; then
	${ECHO} 1>&2 "$self: Cannot write to `cd $fetchdir && pwd`"
	exit 1
fi
@


1.19
log
@Remove clauses 3,4 from TNF-only copyright blocks.

This is based on the decision The NetBSD Foundation made in 2008 to
do so, which was already applied to src.

This change has been applied to code which is likely not in other
repositories.

ok board@@, reviewed by riastradh@@
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.18 2015/09/19 11:45:56 dsainty Exp $
d106 1
a106 1
: ${CHECKSUM:=${PKGSRCDIR}/mk/checksum/checksum}
@


1.18
log
@Alter the behaviour of the "fetch" executable to accept zero download sites
as an acceptable input.

Fixes Pkgsrc for users that set PKG_RESUME_TRANSFERS=yes in their mk.conf if
installing a package that has an interactive fetch stage, after
mk/fetch/fetch.mk revision 1.66.

Tested for all combinations of:

+ PKG_RESUME_TRANSFERS=yes/no; and
+ interactive fetch=yes/no; and
+ distfile=downloaded/not downloaded

... with no signs of misbehaviour.

The specific case that was broken, and this change fixes, is:

+ PKG_RESUME_TRANSFERS=yes; and
+ interactive fetch=yes; and
+ distfile=downloaded

This change was designed with the following considerations:

1. Given it's a freeze, keep it simple.

2. The change in behaviour allows 'fetch' to succeed where it would previously
   fail fatally, so it's unlikely to affect any (intentional) existing
   behaviour in Pkgsrc.

3. The behaviour of 'fetch' with zero sites is essentially the same as a
   fetch where all the possible download sites fail, I.e. Pkgsrc already
   expects to handle such behaviour.

ok gdt@@
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.17 2015/09/08 12:37:56 joerg Exp $
a18 7
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#        This product includes software developed by the NetBSD
#        Foundation, Inc. and its contributors.
# 4. Neither the name of The NetBSD Foundation nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
@


1.17
log
@Make the post-fetch hook a bit more useful by also handing down the URL
the file was obtained from. This makes it easier to skip any local
mirrors.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.16 2015/09/08 11:13:12 joerg Exp $
d46 1
a46 1
#	fetch [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file site ...
d126 1
a126 1
	${ECHO} 1>&2 "usage: $self [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file site ..."
d169 1
a169 1
if ${TEST} $# -lt 2; then
@


1.16
log
@Add a hook for running an action after a successful fetch.

This is intended for keeping distfile archives synchronised between
different machines without requiring NFS mounts or the like.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.15 2013/02/16 07:32:28 obache Exp $
d71 3
a73 2
#		After a successful fetch, run hook with the relative path
#		as argument.
d312 1
a312 1
	"$post_fetch" $path
@


1.15
log
@Remove fetched file if fetch command succeeds to fetch but checksum mismatch
with PKG_RESUME_TRANSFERS=YES.

PR 39896.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.14 2009/08/02 02:38:52 joerg Exp $
d5 1
a5 1
# Copyright (c) 2006 The NetBSD Foundation, Inc.
d46 1
a46 1
#	fetch [-c] [-d dir] [-f distinfo] [-r] [-v] file site ...
d70 4
d125 1
a125 1
	${ECHO} 1>&2 "usage: $self [-c] [-d dir] [-f distinfo] [-r] [-v] file site ..."
d132 1
d140 1
d310 3
@


1.14
log
@Add support for fetching from URLs that do not end in the distfile.
If the URL is prefixed with a dash, it is considered complete.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.13 2009/07/20 09:09:02 tnn Exp $
d249 1
@


1.13
log
@Specify the name of the distfile instead of allowing the fetch tool to
infer it from the URL. No objections on tech-pkg@@
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.12 2008/12/15 09:01:30 taca Exp $
d270 8
d281 1
a281 1
		fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $site$file"
d283 1
a283 1
		fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $site$file ${FETCH_AFTER_ARGS}"
@


1.12
log
@Remove extra debug codes which noted by Ryo HAYASAKA as PR pkg/40175.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.11 2008/12/14 14:09:32 rillig Exp $
d275 1
a275 1
		fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} $site$file ${FETCH_AFTER_ARGS}"
@


1.11
log
@Added a -v flag to the verify_file function, which in case of a
verification failure prints a useful and concrete error message.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.10 2008/09/15 21:07:46 rillig Exp $
a206 2
		echo "foo"
		read line
@


1.10
log
@Corrected the usage message.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.9 2007/08/04 15:00:29 obache Exp $
d192 1
a192 1
# verify_file $file
d200 1
d202 4
a205 1
	${TEST} -f $_file || return 1
d207 6
a212 1
		${CHECKSUM} -s ".pkgsrc.resume" $distinfo ${_file} || return 1
d216 4
a219 1
		${TEST} "$_size" -eq "$distsize" || return 1
d285 1
a285 1
	if verify_file $outputpath; then
a287 2
	else
		${ECHO} 1>&2 "$self: Unable to verify fetched file $file"
@


1.9
log
@Move ${DISTDIR} write permission check to just before really need to write.
Patch provided by Aleksej Saushev in PR 36473.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.8 2007/01/18 10:51:48 rillig Exp $
d46 1
a46 1
#	fetch [-c] [-d dir] [-f distinfo] file site ..."
d121 1
a121 1
	${ECHO} 1>&2 "usage: $self [-c] [-f distinfo] [-r] file site ..."
@


1.8
log
@Backed out my previous change. The variable fetchdir needs to be a
relative path.

The error message nevertheless includes the full path.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.7 2007/01/10 14:31:35 rillig Exp $
a173 6
${TEST} -d $fetchdir || ${MKDIR} -p $fetchdir 2>/dev/null
if ${TEST} ! -w $fetchdir; then
	${ECHO} 1>&2 "$self: Cannot write to `cd $fetchdir && pwd`"
	exit 1
fi

d220 6
@


1.7
log
@It's more informative to use absolute pathnames in error messages than
just ".".
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.6 2006/08/03 14:02:58 jlam Exp $
d127 1
a127 1
fetchdir=.
a144 3

fetchdir=`cd $fetchdir && exec pwd`

d176 1
a176 1
	${ECHO} 1>&2 "$self: Cannot write to $fetchdir"
@


1.6
log
@Typo in comment.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.5 2006/07/20 13:31:08 joerg Exp $
d145 3
@


1.5
log
@verify_file() returns a non-zero result whenever the distfile does not
currently exist. This triggers a bug in the Bourne shell of at least
DragonFly and Irix, when set -e is also active. Remove set -e to avoid
this. Discussed with jlam@@, shown by DragonFly bulk builds, complains on
DragonFly's user list and PR 34036.

A test case for this shell bug is, which should return 0:

set -e

testf () {
	test -f /nonexistent || return 1
	return 0
}

if testf; then
	exit 1
fi
exit 0
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.4 2006/07/19 15:13:40 jlam Exp $
d61 1
a61 1
#		not sucessful.
@


1.4
log
@Comment the code a bit more.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.3 2006/07/19 14:16:29 jlam Exp $
a106 2
set -e		# exit on errors

@


1.3
log
@Add a verbose flag to the fetch script to output the actual fetch command.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.2 2006/07/18 22:41:06 jlam Exp $
d257 3
@


1.2
log
@Overhaul fetch module:

* All the smarts is now encapsulated in the "fetch" script.  The fetch
  script understands how to use the distinfo file (if specified) to
  look up the size and checksums of the file to fetch and will use
  that information to verify checksums of the fetched files or resume
  transfers of interrupted fetches.

* Move the default settings for FETCH_RESUME_ARGS and FETCH_OUTPUT_ARGS
  for "ftp" from mk/defaults/mk.conf into mk/fetch/fetch.mk.  We rewrite
  it to avoid needing conditional statements.

* Avoid spawning a new make(1) process just to mirror a distfile.

* Split out fetch-list targets into a separate file fetch-list.mk.
  These targets should probably be moved into a standalone script.

* Fix distclean target to properly remove partial downloads.
@
text
@d3 1
a3 1
# $NetBSD: fetch,v 1.1 2006/07/17 15:34:22 jlam Exp $
d75 3
d131 1
d138 1
d262 1
a262 1
		${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $site$file
d264 4
a267 2
		${FETCH_CMD} ${FETCH_BEFORE_ARGS} $site$file ${FETCH_AFTER_ARGS}
	  fi )
@


1.1
log
@First cut at a fetch script to replace the humungous fetch "macros" in
fetch.mk.  This script currently completely replaces the functionality
in _FETCH_FILE.  I will eventually add the ability to resume a file
transfer to this script.
@
text
@d3 1
a3 1
# $NetBSD$
d46 1
a46 1
#	fetch [-c] [-f distinfo] file site ..."
d52 1
a52 1
#	If the file cannot be fetched successfully, then try the next
d55 3
d63 2
d70 32
d106 3
a108 1
: ${CHECKSUM:=checksum}
d111 2
a112 1
: ${RM:=rm}
d114 1
d120 1
a120 1
	${ECHO} 1>&2 "usage: $self [-c] [-f distinfo] file site ..."
d126 2
d131 1
d133 1
d143 1
a143 1
	${ECHO} 1>&2 "$self: \`\`-c'' requires \``-f distinfo''"
d146 10
a155 3
if ${TEST} -n "$resume" -a ! -z "$distinfo"; then
	${ECHO} 1>&2 "$self: \`\`-r'' requires \``-f distinfo''"
	exit 1
d164 1
d171 3
a173 4
dir="${file%/*}"
${TEST} "$dir" != "$file" || dir=.
if ${TEST} ! -w "$dir/."; then
	${ECHO} 1>&2: "$self: Cannot write to $dir"
d186 4
a189 1
		${TEST} "d_file" != "($file)" || continue
d195 17
a211 4
checksum_file() {
	_file="$1"
	if ${TEST} -z "$checksum" || ${CHECKSUM} $distinfo $_file; then
		return 0;
d213 1
a213 1
	return 1
d216 36
a251 1
bfile="${file##*/}"
d254 14
a267 4
	${TEST} -z "$distsize" || ${ECHO} "=> [$distsize $distunits]"
	fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${site}${bfile} ${FETCH_AFTER_ARGS}"
	if $fetch_cmd; then
		: # successful fetch of file
d269 1
a269 1
		${RM} -f $file
d271 3
a273 3
	if ${TEST} -f $file; then
		if checksum_file $file; then
			break
d275 1
a275 2
			${ECHO} 1>&2 "$self: Checksum failure for $bfile"
			${RM} -f $file
a276 2
	else
		${ECHO} 1>&2 "$self: Unable to fetch expected file $bfile"
d279 1
a279 1
if ${TEST} -f "$file"; then
@