diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/ChangeLog-roam.txt ./ChangeLog-roam.txt --- ../../branches/vendor/bomstrip/ChangeLog-roam.txt 1970-01-01 02:00:00.000000000 +0200 +++ ./ChangeLog-roam.txt 2008-10-20 14:28:58.000000000 +0300 @@ -0,0 +1,88 @@ +This is a log of the changes made in the bomstrip-roam patchsets. + +bomstrip-9-roam-02 2008/10/20 + - revert the signed/unsigned bugfix from bomstrip-8-roam-01 + - fix the problem correctly - check for error codes from + the fread() and fwrite() invocations properly, as "suggested" + by the Debian GNU/Linux build hardening tools + +bomstrip-9-roam-01 2008/06/26 + - rename bomstrip-2.pl to bomstrip2.pl in the Makefile + +bomstrip-8-roam-06 2008/06/16 + - bomstrip.cpp + - include for exit(3) + +bomstrip-8-roam-05 2008/06/16 + - Makefile + - actually install the GNU Forth implementation + +bomstrip-8-roam-04 2008/06/16 + - bomstrip.1 + - list the authors of the bomstrip implementations + - bomstrip.fs + - add a GNU Forth implementation + +bomstrip-8-roam-03 2008/06/14 + - start this ChangeLog :) + - Makefile + - install the bomstrip.awk and bomstrip-2.pl implementations + - bomstrip.awk + - add an awk implementation that fails on files with EOL on + the last line + - bomstrip-2.pl + - add a one-liner Perl implementation + +bomstrip-8-roam-02 2008/06/13 + - actually install the C++ implementation of bomstrip that I added + in the roam-01 patch :) + +bomstrip-8-roam-01 2008/06/13 + - the first bomstrip-roam patchset + - bomstrip.c + - fix a signed/unsigned (ssize_t) bug + - make the buffer global so that it is always initialized with + zeroes, thus making sure strcmp() will always work and also being + a bit nicer to the main() function's stack + - clean up the code a bit, fixing all C compiler warnings + - bomstrip.cpp + - add a C++ implementation + - bomstrip.pl + - do what the author seems to have meant - the input record + separator is in the $/ special variable, not $! :) + Still, bomscript.pl worked previously, too, because the "print" + function evaluates its arguments in list context, and it still + read the whole of the contents, albeit somewhat + inefficiently, line by line :) + - bomstrip.py + - remove the diagnostic output, since it messed up the stripped + data :) + - bomstrip-files.sh + - add a Bourne shell script to strip files in-place + - bomstrip.1 + - add a manual page, hopefully describing all implementations :) + - also describe the bomstrip-files utility + - Makefile + - add a Makefile + - install the implementation sources and the tests into + share/bomstrip/ + - install the bomstrip and bomstrip-files utilities + - install the bomstrip(1) manual page + - bom-ml1, bom-ml2, bom-ml3, bom-ml4, + nobom-ml1, nobom-ml2, nobom-ml3, nobom-ml4, + correct/rbom-ml1, correct/rbom-ml2, correct/rbom-ml3, + correct/rbom-ml4, + correct/rnobom-ml1, correct/rnobom-ml2, correct/rnobom-ml3, + correct/rnobom-ml4 + - add a couple of tests: + - multi-line messages + - no line-feed character on the last line + - DOS-style CR/LF line endings + - test.sh + - make this a bit more efficient with a loop on the files + - make the test command and the test set configurable + - exit with a non-zero result code if any of the tests fails, + but only after running all the tests + - add the new tests + +Comments: Peter Pentchev diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/Makefile ./Makefile --- ../../branches/vendor/bomstrip/Makefile 1970-01-01 02:00:00.000000000 +0200 +++ ./Makefile 2008-06-26 11:05:17.000000000 +0300 @@ -0,0 +1,108 @@ +# Makefile for bomstrip, the UTF-8 BOM stripper +# +# Written by Peter Pentchev in 2008. +# This file is hereby placed into the public domain. + +LOCALBASE?= /usr/local +PREFIX?= ${LOCALBASE} +BINDIR?= ${PREFIX}/bin +MANDIR?= ${PREFIX}/man/man +SHAREDIR?= ${PREFIX}/share/bomstrip + +PROG= bomstrip +SRCS= bomstrip.c +OBJS= ${SRCS:%.c=%.o} +BFILES= bomstrip-files + +SOURCES= bomstrip.awk \ + bomstrip.b \ + bomstrip.bf \ + bomstrip.c \ + bomstrip.cpp \ + bomstrip.fs \ + bomstrip.hs \ + bomstrip.ocaml \ + bomstrip.ook \ + bomstrip.pas \ + bomstrip.php \ + bomstrip.pl \ + bomstrip2.pl \ + bomstrip.ps \ + bomstrip.py \ + bomstrip.rb \ + bomstrip.sed \ + bomstrip.unl +TESTS= bom0 bom1 bom-ml1 bom-ml2 bom-ml3 bom-ml4 \ + nobom0 nobom1 nobom2 nobom-ml1 nobom-ml2 nobom-ml3 nobom-ml4 +TESTS_CORRECT= ${TESTS:%=correct/r%} + +CFLAGS_WARN?= -Wall -W -Werror -ansi -pedantic -Wbad-function-cast \ + -Wcast-align -Wcast-qual -Wchar-subscripts -Winline \ + -Wmissing-prototypes -Wnested-externs -Wpointer-arith \ + -Wredundant-decls -Wshadow -Wstrict-prototypes -Wwrite-strings +CFLAGS_OPT?= -O2 -pipe +CFLAGS_DBG?= + +CFLAGS?= ${CFLAGS_WARN} ${CFLAGS_OPT} ${CFLAGS_DBG} +LDFLAGS?= +LIBS?= + +COPY?= -c +STRIP?= -s + +BINOWN?= root +BINGRP?= root +BINMODE?= 755 + +SHAREOWN?= ${BINOWN} +SHAREGRP?= ${BINGRP} +SHAREMODE?= 644 + +CC?= cc +GZIP?= gzip -c -9 +LN?= ln +MKDIR?= mkdir -p +RM?= rm -f + +INSTALL?= install +INSTALL_PROGRAM?= ${INSTALL} ${COPY} ${STRIP} -o ${BINOWN} -g ${BINGRP} -m ${BINMODE} +INSTALL_SCRIPT?= ${INSTALL} ${COPY} -o ${BINOWN} -g ${BINGRP} -m ${BINMODE} +INSTALL_DATA?= ${INSTALL} ${COPY} -o ${SHAREOWN} -g ${SHAREGRP} -m ${SHAREMODE} + +all: ${PROG} ${PROG}.1.gz + +clean: + ${RM} ${PROG} ${OBJS} ${PROG}.1.gz + +install: all + ${MKDIR} ${DESTDIR}${BINDIR} + ${INSTALL_PROGRAM} ${PROG} ${DESTDIR}${BINDIR} + ${INSTALL_SCRIPT} ${BFILES}.sh ${DESTDIR}${BINDIR}/${BFILES} + ${MKDIR} ${DESTDIR}${MANDIR}1 + ${INSTALL_DATA} ${PROG}.1.gz ${DESTDIR}${MANDIR}1 + ${RM} ${DESTDIR}${MANDIR}1/${BFILES}.1.gz + ${LN} ${DESTDIR}${MANDIR}1/${PROG}.1.gz ${DESTDIR}${MANDIR}1/${BFILES}.1.gz + ${MKDIR} ${DESTDIR}${SHAREDIR}/source + ${INSTALL_DATA} ${SOURCES} ${DESTDIR}${SHAREDIR}/source + ${MKDIR} ${DESTDIR}${SHAREDIR}/tests/correct + ${INSTALL_DATA} ${TESTS} ${DESTDIR}${SHAREDIR}/tests + ${INSTALL_DATA} ${TESTS_CORRECT} ${DESTDIR}${SHAREDIR}/tests/correct + +# If you set the TEST_X environment variable to, say, "-x", you may +# get a bit more verbose shell output. +# For the present, the test.sh script does not take any arguments, but +# why not foresee the possibility - just set TEST_ARGS and go. +# +test: all + BOM=./bomstrip BOMFILES="${TESTS}" sh ${TEST_X} test.sh ${TEST_ARGS} + +${PROG}: ${OBJS} + ${CC} ${LDFLAGS} ${STRIP} -o ${PROG} ${OBJS} ${LIBS} + +.c.o: + ${CC} -c ${CFLAGS} $< + +%.1.gz: %.1 + ${GZIP} $< > $@ + +.PHONY: all clean install test diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/bomstrip-files.sh ./bomstrip-files.sh --- ../../branches/vendor/bomstrip/bomstrip-files.sh 1970-01-01 02:00:00.000000000 +0200 +++ ./bomstrip-files.sh 2008-06-26 11:05:17.000000000 +0300 @@ -0,0 +1,19 @@ +#!/bin/sh +# +# Written by Peter Pentchev in 2008. +# This file is hereby placed into the public domain. + +set -e + +[ -z "$BOMSTRIP" ] && BOMSTRIP='bomstrip' + +if [ "$#" -eq 0 ]; then + echo 'Usage: bomstrip-files file...' 1>&2 + exit 1 +fi + +while [ "$#" -ne 0 ]; do + cp "$1" "$1.bom" + $BOMSTRIP < "$1.bom" > "$1" + shift +done diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/bomstrip.1 ./bomstrip.1 --- ../../branches/vendor/bomstrip/bomstrip.1 1970-01-01 02:00:00.000000000 +0200 +++ ./bomstrip.1 2008-06-26 11:05:17.000000000 +0300 @@ -0,0 +1,83 @@ +.\" Written by Peter Pentchev in 2008. +.\" This file is hereby placed into the public domain. +.\" +.Dd June 14, 2008 +.Dt BOMSTRIP 1 +.Os +.Sh NAME +.Nm bomstrip , +.Nm bomstrip-files +.Nd strip the BOM sequence from UTF-8 files +.Sh SYNOPSIS +.Nm +.Nm bomstrip-files +.Ar +.Sh DESCRIPTION +The +.Nm +utility reads UTF-8 data from its standard input and copies it to +its standard output, stripping the BOM (byte-order mark) from +the beginning of the text if it is present. +There are no command-line options and no parameters. +.Pp +The +.Nm bomstrip-files +utility removes the UTF-8 BOM from the specified files, saving each +file's original contents with a +.Pa .bom +extension. +It uses the +.Nm +utility, trying to execute it as +.Dq Nm ; +if the +.Nm +utility is installed under another name, or if a more complex command +is desired, it may be supplied in the +.Ev BOMSTRIP +environment variable. +.Sh EXAMPLES +Strip the BOM, if present, from a text file: +.Pp +.Dl bomstrip < bom.txt > nobom.txt +.Pp +Strip the BOM, if present, from all text files, backing them up with a +.Pa .bom +extension: +.Pp +.Dl bomstrip-files *.txt +.Pp +Use the OCAML implementation of +.Nm : +.Pp +.Dl env BOMSTRIP='ocaml bomstrip.ocaml' bomstrip-files *.txt +.Sh SEE ALSO +The +.Nm +home page: +.Li http://www.xs4all.nl/~mechiel/projects/bomstrip/ +.Sh HISTORY +The +.Nm +utility (in many languages) was written by +.An Mechiel Lukkien , +with implementations in various languages sent to him by others, including +.An Andreas Gohr , +.An Andrew Gerrand , +.An Berteun Damman , +.An Matthijs Bomhoff , +.An Peter Pentchev , +and +.An Ruben Smelik . +The +.Nm bomstrip-files +utility and this manual page were written by +.An Peter Pentchev +in the hope that they reflect the behavior of all the +.Nm +implementations in all languages. +.Sh AUTHORS +.An Mechiel Lukkien +.Aq mechiel@xs4all.nl +.An Peter Pentchev +.Aq roam@ringlet.net diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/bomstrip.c ./bomstrip.c --- ../../branches/vendor/bomstrip/bomstrip.c 2008-06-25 17:53:09.000000000 +0300 +++ ./bomstrip.c 2008-10-20 14:22:12.000000000 +0300 @@ -3,19 +3,22 @@ #include #include -void -usage(char *prog) +char buf[BUFSIZ]; +const char *utf8bom = "\xef\xbb\xbf"; + +static void usage(const char *); + +static void +usage(const char *prog) { fprintf(stderr, "usage: %s\n", prog); exit(1); } int -main(int argc, char *argv[]) +main(int argc, const char * const argv[]) { size_t nread; - char buf[65536]; - char *utf8bom = "\xef\xbb\xbf"; if (argc > 1) usage(argv[0]); @@ -24,14 +27,14 @@ if (nread == 0) return 0; if (strcmp(buf, utf8bom) != 0) - fwrite(buf, 1, nread, stdout); + if (fwrite(buf, 1, nread, stdout) < nread) + exit(1); for (;;) { nread = fread(buf, 1, sizeof buf, stdin); - if (nread < 0) - exit(1); if (nread == 0) - return 0; - fwrite(buf, 1, nread, stdout); + return ferror(stdin) && 1; + if (fwrite(buf, 1, nread, stdout) < nread) + exit(1); } return 0; } diff -urN -x .svn -x '*bom-ml*' ../../branches/vendor/bomstrip/test.sh ./test.sh --- ../../branches/vendor/bomstrip/test.sh 2008-06-25 17:53:09.000000000 +0300 +++ ./test.sh 2008-06-26 11:05:17.000000000 +0300 @@ -1,7 +1,8 @@ #!/bin/sh test -z "$BOM" && BOM='ocaml ./bomstrip.ocaml' -test -z "$BOMFILES" && BOMFILES='bom0 bom1 nobom0 nobom1 nobom2' +test -z "$BOMFILES" && BOMFILES='bom0 bom1 bom-ml1 bom-ml2 bom-ml3 bom-ml4 '\ +'nobom0 nobom1 nobom2 nobom-ml1 nobom-ml2 nobom-ml3 nobom-ml4' res=0 for f in $BOMFILES; do