head 1.10; access; symbols pkgsrc-2017Q2:1.8.0.6 pkgsrc-2017Q2-base:1.8 pkgsrc-2017Q1:1.8.0.4 pkgsrc-2017Q1-base:1.8 pkgsrc-2016Q4:1.8.0.2 pkgsrc-2016Q4-base:1.8 pkgsrc-2016Q3:1.7.0.12 pkgsrc-2016Q3-base:1.7 pkgsrc-2016Q2:1.7.0.10 pkgsrc-2016Q2-base:1.7 pkgsrc-2016Q1:1.7.0.8 pkgsrc-2016Q1-base:1.7 pkgsrc-2015Q4:1.7.0.6 pkgsrc-2015Q4-base:1.7 pkgsrc-2015Q3:1.7.0.4 pkgsrc-2015Q3-base:1.7 pkgsrc-2015Q2:1.7.0.2 pkgsrc-2015Q2-base:1.7 pkgsrc-2015Q1:1.6.0.6 pkgsrc-2015Q1-base:1.6 pkgsrc-2014Q4:1.6.0.4 pkgsrc-2014Q4-base:1.6 pkgsrc-2014Q3:1.6.0.2 pkgsrc-2014Q3-base:1.6 pkgsrc-2014Q2:1.5.0.4 pkgsrc-2014Q2-base:1.5 pkgsrc-2014Q1:1.5.0.2 pkgsrc-2014Q1-base:1.5 pkgsrc-2013Q4:1.4.0.6 pkgsrc-2013Q4-base:1.4 pkgsrc-2013Q3:1.4.0.4 pkgsrc-2013Q3-base:1.4 pkgsrc-2013Q2:1.4.0.2 pkgsrc-2013Q2-base:1.4 pkgsrc-2013Q1:1.3.0.10 pkgsrc-2013Q1-base:1.3 pkgsrc-2012Q4:1.3.0.8 pkgsrc-2012Q4-base:1.3 pkgsrc-2012Q3:1.3.0.6 pkgsrc-2012Q3-base:1.3 pkgsrc-2012Q2:1.3.0.4 pkgsrc-2012Q2-base:1.3 pkgsrc-2012Q1:1.3.0.2 pkgsrc-2012Q1-base:1.3 pkgsrc-2011Q4:1.2.0.20 pkgsrc-2011Q4-base:1.2 pkgsrc-2011Q3:1.2.0.18 pkgsrc-2011Q3-base:1.2 pkgsrc-2011Q2:1.2.0.16 pkgsrc-2011Q2-base:1.2 pkgsrc-2011Q1:1.2.0.14 pkgsrc-2011Q1-base:1.2 pkgsrc-2010Q4:1.2.0.12 pkgsrc-2010Q4-base:1.2 pkgsrc-2010Q3:1.2.0.10 pkgsrc-2010Q3-base:1.2 pkgsrc-2010Q2:1.2.0.8 pkgsrc-2010Q2-base:1.2 pkgsrc-2010Q1:1.2.0.6 pkgsrc-2010Q1-base:1.2 pkgsrc-2009Q4:1.2.0.4 pkgsrc-2009Q4-base:1.2 pkgsrc-2009Q3:1.2.0.2 pkgsrc-2009Q3-base:1.2 pkgsrc-2009Q2:1.1.1.1.0.10 pkgsrc-2009Q2-base:1.1.1.1 pkgsrc-2009Q1:1.1.1.1.0.8 pkgsrc-2009Q1-base:1.1.1.1 pkgsrc-2008Q4:1.1.1.1.0.6 pkgsrc-2008Q4-base:1.1.1.1 pkgsrc-2008Q3:1.1.1.1.0.4 pkgsrc-2008Q3-base:1.1.1.1 cube-native-xorg:1.1.1.1.0.2 cube-native-xorg-base:1.1.1.1 pkgsrc-base:1.1.1.1 TNF:1.1.1; locks; strict; comment @# @; 1.10 date 2017.07.09.22.31.23; author schmonz; state dead; branches; next 1.9; commitid UxpF2XFZ6GSMABYz; 1.9 date 2017.07.09.22.27.47; author schmonz; state Exp; branches; next 1.8; commitid HA8XB8AVuxnBzBYz; 1.8 date 2016.11.07.13.02.45; author schmonz; state Exp; branches; next 1.7; commitid vo3D5Pk0cj62ictz; 1.7 date 2015.05.23.18.21.16; author schmonz; state Exp; branches; next 1.6; commitid OSu4Hcyx41evdBmy; 1.6 date 2014.07.06.15.21.32; author schmonz; state Exp; branches; next 1.5; commitid u8JqTqtGdmtzxkHx; 1.5 date 2014.02.20.19.15.44; author schmonz; state Exp; branches; next 1.4; commitid wuOclrRpY4PWcSpx; 1.4 date 2013.06.04.21.28.26; author schmonz; state Exp; branches; next 1.3; commitid zcxj2ibpPUwCklSw; 1.3 date 2012.01.10.01.04.00; author schmonz; state Exp; branches; next 1.2; 1.2 date 2009.08.27.13.22.42; author schmonz; state Exp; branches; next 1.1; 1.1 date 2008.07.26.23.37.29; author schmonz; state Exp; branches 1.1.1.1; next ; 1.1.1.1 date 2008.07.26.23.37.29; author schmonz; state Exp; branches; next ; desc @@ 1.10 log @Normalize patch filenames. No functional change. @ text @$NetBSD: patch-aa,v 1.9 2017/07/09 22:27:47 schmonz Exp $ Call it "xapian-omega" and follow hier(7). --- Makefile.in.orig 2017-04-24 02:19:50.000000000 +0000 +++ Makefile.in @@@@ -92,7 +92,7 @@@@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @@build@@ host_triplet = @@host@@ -pkglibbin_PROGRAMS = omega$(EXEEXT) +pkglibbin_PROGRAMS = xapian-omega$(EXEEXT) bin_PROGRAMS = omindex$(EXEEXT) scriptindex$(EXEEXT) \ omindex-list$(EXEEXT) check_PROGRAMS = atomparsetest$(EXEEXT) csvesctest$(EXEEXT) \ @@@@ -791,7 +791,7 @@@@ AM_CPPFLAGS = \ -DCONFIGFILE_SYSTEM=\"$(sysconfdir)/omega.conf\" \ -DPKGLIBBINDIR=\"$(pkglibbindir)\" -pkglibbindir = $(pkglibdir)/bin +pkglibbindir = $(libexecdir)/cgi-bin dist_pkglibbin_SCRIPTS = mhtml2html outlookmsg2html rfc822tohtml vcard2text dist_bin_SCRIPTS = dbi2omega htdig2omega mbox2omega dist_check_SCRIPTS = omegatest @@@@ -804,7 +804,7 @@@@ TESTS = atomparsetest$(EXEEXT)\ urlenctest$(EXEEXT)\ utf8converttest$(EXEEXT) -omegadatadir = $(datadir)/omega +omegadatadir = $(datadir)/xapian-omega dist_omegadata_DATA = htdig2omega.script mbox2omega.script dist_sysconf_DATA = omega.conf @@@@ -1099,8 +1099,8 @@@@ md5test$(EXEEXT): $(md5test_OBJECTS) $(m common/str.$(OBJEXT): common/$(am__dirstamp) \ common/$(DEPDIR)/$(am__dirstamp) -omega$(EXEEXT): $(omega_OBJECTS) $(omega_DEPENDENCIES) $(EXTRA_omega_DEPENDENCIES) - @@rm -f omega$(EXEEXT) +xapian-omega$(EXEEXT): $(omega_OBJECTS) $(omega_DEPENDENCIES) $(EXTRA_omega_DEPENDENCIES) + @@rm -f xapian-omega$(EXEEXT) $(AM_V_CXXLD)$(CXXLINK) $(omega_OBJECTS) $(omega_LDADD) $(LIBS) common/getopt.$(OBJEXT): common/$(am__dirstamp) \ common/$(DEPDIR)/$(am__dirstamp) @ 1.9 log @Update to 1.4.4. From the changelog: indexers: * omindex: + 1.4.3 added a new --sample option, but contrary to the documentation the default behaviour was to take the sample from the meta description (which was the hard-wired behaviour in 1.4.2 and earlier). The default has now been changed to take the sample from the body. + Index .shtm, .xhtml and .xhtm as HTML by default - .shtm is another extension used for server-parsed HTML (in addition to the more common .shtml), and .xhtm and .xhtml are XHTML. + Fix fallback lookup for extension containing upper case. User mappings worked, but built-in extension to MIME type mappings were effectively being ignored (because the result of the function call was not being checked). Bug introduced in 1.3.4. + Fix term-based date ranges, broken by changes in 1.4.2. Found and diagnosed by Gaurav Arora. + Handle date range with start after end better - with term-based ranges, this used to generate a bogus filter, but now just generates Dlatest. + Use Y-term when range starts/ends at year start/end. Previously we used 12 M-terms for these cases. + Use full leap-year check when constructing term-based date ranges - previous code was good until 2100, but even then it would only result in an extra term being included for a non-existent February 29th in rare cases. + Add support for indexing vCard files if Perl and its Text::vCard module are available. + Recognise application/x-rpm as alternative type since libmagic reports this rather than application/x-redhat-package-manager. + Use official MIME type application/vnd.debian.binary-package for debian packages. We used to map .deb and .udeb to application/x-debian-package, but in 2014 (after we added that support for .deb) an official type was registered with IANA. We now map extensions .deb and .udeb to the official type, but the unofficial type is still recognised (older versions of libmagic probably report it, and users may be mapping to it). + Handle PHP as MIME type text/x-php. The main difference this makes is that PHP files which don't have extension '.php' (e.g. .phtml, .phps, .php5, .ph4, etc) get identified by libmagic as text/x-php and will now be indexed. It also means that the user can now more easily configure different filters for HTML and PHP. + Don't use meta description as sample by default. Now we have dynamic snippets (via $snippet), the body text is a better default. Also generated HTML sometimes has unhelpful content in the meta description. To get the previous behaviour, use the new omindex command line option: --sample=description omega: * New OmegaScript command $cgiparams which returns a list of the parameter names. * Handle tab in a CGI parameter name in the same way as space. Mostly this is a way to avoid having tabs in CGI parameter names - they aren't useful, but if they could have tabs in we can't put CGI parameter names in a list. templates: * query: Fix highlighting of matching terms. We were using both $snippet and $highlight, which results in double highlighting and HTML escaping, most noticeable by literal and appearing around matching terms in the rendered HTML snippet. Reported by Mark Thomas on xapian-discuss. build system: * If gen-mimemap failed after creating mimemap.h, the rule wouldn't get rerun. @ text @d1 1 a1 1 $NetBSD: patch-aa,v 1.8 2016/11/07 13:02:45 schmonz Exp $ @ 1.8 log @Update to 1.4.1. From the changelog: omindex: + Also index leafname with _ and & replaced by spaces. Literal spaces are often avoided in filenames, and "hello_world.txt" ought to be searchable for via "hello" and "world". Partly addresses #618, reported by Julien Pfefferkorn. + Make named entity look-up (e.g. é -> 233) use the same keyword-lookup table approach we already use for HTML tags and built-in MIME content-types, rather than a std::map, which makes it faster while using less memory. + Avoid using the shell to run most external commands as it's unnecessary overhead. For the built-in filters, the only cases which now use a shell are where we run two unzip commands. For user-specified commands, a simple and slightly conservative test is used, which should avoid a shell in most common cases where it isn't needed. Notably, environment variables set before the command are handled. + Track files which couldn't be indexed in the user metadata and skip them by default on subsequent runs to avoid the costs of repeatedly running a filter on a file it can't handle. Run omindex with --retry-failed to retry such files. + Overhaul the "per-site" terms: - 'H' prefix is hostname as before, except that if the term would be > 240 bytes (unlikely but possible) the end is hashed is the same way 'U' prefix terms are. - 'P' terms are now added for every directory level, not just the start URL's path. - A new 'J' prefix term is added with the start URL (less any trailing '/'), which means all files indexed from a particular "site" are now indexed by one term. See #376. + Add 'skip' pseudo-mimetype which extensions can be mapped to, and they will then be reported and skipped (to complement the existing 'ignore' pseudo-mimetype which causes files with the specified extension to be quietly ignored). + Treat a command of 'true' specially as meaning make the text extraction a no-op (as actually running /bin/true effectively would). This provides a way to index some file types by only meta-data. Fixes #519, reported by Brian Burton. + Add support for wildcard mimetypes */* and *. Combined with filter command ``true`` for indexing by meta-data only, you can specify a fall back case of indexing by meta-data only using ``--filter '*:true'``. From a suggestion by Brian Burton on xapian-discuss. + Index message/rfc822 and message/news. These are individually saved email messages and news articles. + Index archived web page formats MAFF and MHTML. + Handle .xla, yet another XL extension. + Handle metadata in LibreOffice HTML export (dcterms.subject, dcterms.description, dcterms.creator and dcterms.contributor). + Use zlib's gzopen() instead of invoking "gzip -dc" for compressed Abiword documents. + Add support for %f in command passed to --filter to allow specifying commands where the input file is not the final argument. Fixed #570, reported by Charles Atkinson. + Allow --filter to handle commands which produce output in a temporary file rather than on stdout. + Allow --filter to specify the character set of the output the filter produces. + Handle application/vnd.ms-excel, text/x-perl and application/x-dvi via default --filter settings instead of hardcoded cases (now possible thanks to the new abilities that --filter has). + Add support for specifying a MIME subtype of '*' in --filter arguments. + Add -track-ctime option to allow omindex to pick up changes to file ownership and permissions. + Index terms from the leafname with an 'F' prefix, rather than treating them as more body text. (Fixes #633, reported by Emmanuel Garette) + The starting URL wasn't previously URL encoded. In 1.2.18, a minimally intrusive fix was implemented. In 1.3.2, we now encode the starting URL as we do for the rest of the filename. + Don't assume .doc is application/msword but let libmagic decide, since .doc files may actually be RTF, and sometimes people use .doc for plain-text documentation. + Add support for indexing 'topic' and 'created date' meta-data for OpenDocument format and HTML. + Index "topic" for PDF documents. + Commit changes and exit, rather than skipping the current file on most unexpected errors reading directories or initialising libmagic - otherwise we can end up deleting a lot of database entries on errors like EHOSTDOWN when indexing network mounts. + Add --opendir-sleep=SECS option to allow working around problems with indexing files on Microsoft DFS shares. + If we get ENOTDIR trying to index a file, skip it quietly (unless in verbose mode) as we already do if we get ENOENT, since ENOTDIR is what we get if the file and the directory it was in got removed between us getting the filename and trying to open it. + Handle ENOENT, ENOTDIR and EACCES from readdir(). + If we've already opened the file (as we often will have if using a modern libmagic with magic_descriptor() available), then use fstat() on that fd rather than stat()/lstat() on the pathname. + Pass error message string and errno value in ReadError exceptions. + Report strerror(errno) if we can't read a file. + Filtering via text/html now handles HTML documents which specify a charset. + Add support for indexing Microsoft Publisher files using pub2xhtml. + Restrict the length of what we consider to be an extension, currently to 7 characters or whatever the longest extension in the mime_map is if it is longer. + Avoid '//' in temporary filenames (cosmetic only). + Extend --filter to handle commands which produce HTML on stdout. + Don't report an error if a file is deleted (or renamed) between us reading the directory entry for it and trying to read the file itself by default. In --verbose mode, the situation is still reported, but now with a specific message. + If omindex receives any of the signals SIGHUP, SIGINT, SIGQUIT or SIGTERM, then kill any active external filter child process, then handle the signal as we did before. If setpgid() is available, put each external filter in its own process group and kill the whole process group when we get a signal. + Use magic_descriptor() if the version of libmagic we're building against is new enough to have it. This eliminates an extra opening of a file being indexed in certain cases. + Use rst2html to handle .rst and .rest files. + Index title with an 'S' prefix rather than no prefix. + If the document with the highest existing docid before the run was updated, we were reporting it as "added", but now we correctly report it as "updated". + Catch and report std::exception explicitly, so failing to allocate memory is no longer reported as "Unknown exception". omindex-list: New tool to list URLs of all the documents in a database (or list of databases) indexed by omindex. * The HTML parser now explicitly handles , and . * Use a generated compact and efficient table to convert HTML tag names to enum codes - this is both faster and smaller than the approach we were using, with the benefit that the table is auto-generated. * Always use our built-in conversion code for the character sets it can handle (previously we'd use iconv if available; now we only use iconv for other character sets). This gives us more consistent results, and in particular means we now handle BOMs better (at least when using GNU iconv). * A lot of data labelled as "iso-8859-1" is actually "windows-1252". The two only differ in characters which are control characters in iso-8859-1, so assume the latter when we see the former. scriptindex: + Remove special error handling case noting that index=nopos was replaced with indexnopos - this was removed in 1.1.0 so there's been enough time to upgrade. omega: * Add support for sorting by more than one value - e.g. SORT=+1,-2 * Add $msizelower and $msizeupper which provide access to the lower and upper bounds on the number of matches. * Add support for $set{weighting,coord}. * Add weightingpurefilter option. Normally a query consisting only of filter terms won't have relevance weights calculated. This new option allows you to specify a weighting scheme to use for such queries, with the same values supported as for the existing weighting option. For example, $set{weightingpurefilter,coord} will weight such queries by how many filter terms match each document. * $filters now includes DATEVALUE, which means we'll force the first page when reloading or changing page starting from existing URLs upon upgrade to 1.4.1, but the exact same existing URL could be for a search without the date filter where we want to force the first page, so there's an inherent ambiguity there. Forcing first page in this case seems the least problematic side-effect. * Implement $match command for omegascript. Patch from Richhiey Thomas. * Add optional prefix argument to $terms. * $snippet now uses MSet::snippet() instead of the Snipper class. * Add $contains{STRING1,STRING2}. Contributed by Ayush Gupta. * Add support for negated boolean filter terms, specified by CGI parameter "N". * Support a direction prefix on SORT: '+' for ascending, '-' for descending. SORTREVERSE set to non-0 now flips the direction. Fixes #697, reported by Andy Chilton. * Add options argument to $transform. * Cache compiled regexps used in $transform. * Add $ord OmegaScript command which returns the Unicode codepoint for the first character of a UTF-8 string. * Add $chr OmegaScript command which returns the UTF-8 string for given Unicode codepoint. * Add $csv OmegaScript command which escapes a string for use as a field in a CSV file ("always quote" mode inspired by patch from Gaurav Arora.) * New $filters encoding which avoids collisions. We also compare CGI parameter xFILTERS to what $filters would have returned in previous releases, so that on upgrades old format serialised filters are handled correctly. * Fix $jsonarray not to prepend ']' to the first array element. * Skip weighting scheme setup for a pure date range query - it won't be weighted anyway, so we can avoid having to parse weighting scheme parameters, etc. * Use value ranges when date range filtering by value. Should be more efficient than a MatchDecider, and will automatically take advantage of any future value range optimisations in xapian-core. * Add default_db and default_template config options. These allow the default template and default database name to be set via the config file, rather than being stuck with the respective defaults of "default" and "query". Fixes #310, reported by Marco Hennigs. * Add support for non-exclusive filters. Fixes #234, reported by Thomas Viehmann. * Fix handling of multiple P. fields - previously only the first seen was used. These fields are also now taken into account when deciding if the query has changed. $query now returns an OmegaScript list with one entry for each CGI parameter passed. * Allow setting query expansion scheme to "bo1". * Make the $json and $jsonarray force the text to be valid UTF-8, since otherwise the output isn't valid JSON. * Check parameters to $set{weighting,bm25 ...} and $set{weighting,trad ...} converted OK. Based on patch from Aarsh Shah. * Add support to $set{weighting,...} for bb2, dlh, dph, ifb2, ineb2, inl2, lm, pl2 when we're built against a xapian-core which is new enough to have these schemes. * Add $snippet to generate a snippet of text tailored to the search. * Add new $json and $jsonarray OmegaScript commands to support producing JSON output. * Add $truncate command which truncates a string after a word. * Add support for $set{weighting,tfidf} to allow the new TfIdfWeight weighting scheme to be used. + DEFAULTOP now defaults to AND rather than OR, since that matches what pretty much every search engine does these days. Closes ticket#512. * Allow mapping a query string prefix to more than one term prefix (which xapian-core has supported since 1.0.4). * Add support for search inputs for multiple probabilistic prefixes, with support for per-prefix stemmers. * Drop legacy support for handling '.' separated terms in xP - that changed in Omega 0.9.7, more than 5 years ago now. * Remove support for OLDP CGI parameter which was superseded by xP approximately a decade ago, and isn't even documented! * Drop special handling for R-prefixed terms in $prettyterm - we stopped generating these in Xapian 1.0. templates: * Lower case all HTML tags, attributes and values; explicitly close