Set source encoding for template files (#804)

* Add encoding property.

* Apply encoding to FilteredFile (control and maintainer scripts).

* Apply encoding to remaining control files.

* Apply encoding to changes file.

* Hard code ls mapper to UTF8.

* Update docs.

* Add some additional tests to cover custom control files.

* Fix custom open and close token not applied to FilteredFile.

docs/ant.md

@@ -1,18 +1,19 @@
 # How to use jdeb with Ant
 
-Attribute      | Description                                                                  | Required
--------------- | ---------------------------------------------------------------------------- | --------------------------
-destfile       | The debian package to be generated                                           | Yes
-control        | The directory containing the control files                                   | Yes
-compression    | Compression method for the data file (`gzip`, `bzip2`, `xz` or `none`)       | No; defaults to `gzip`
-verbose        | Print detailed info during the package generation                            | No; defaults to `false`
-keyring        | The file containing the PGP keys                                             | No
-key            | The name of the key to be used in the keyring                                | No
-passphrase     | The passphrase to use the key                                                | No
-changesIn      | The changes to add                                                           | No
-changesOut     | The changes file generated                                                   | No
-changesSave    | The merged changes file                                                      | No
-changesEnabled | Enable the creation of the changes file                                      | No; defaults to `true`
+Attribute      | Description                                                                    | Required
+-------------- | ------------------------------------------------------------------------------ | --------------------------
+destfile       | The debian package to be generated                                             | Yes
+control        | The directory containing the control files                                     | Yes
+compression    | Compression method for the data file (`gzip`, `bzip2`, `xz` or `none`)         | No; defaults to `gzip`
+verbose        | Print detailed info during the package generation                              | No; defaults to `false`
+keyring        | The file containing the PGP keys                                               | No
+key            | The name of the key to be used in the keyring                                  | No
+passphrase     | The passphrase to use the key                                                  | No
+changesIn      | The changes to add                                                             | No
+changesOut     | The changes file generated                                                     | No
+changesSave    | The merged changes file                                                        | No
+changesEnabled | Enable the creation of the changes file                                        | No; defaults to `true`
+encoding       | The character character encoding to use when reading control and changes files | No; defaults to default charset of the JVM
 
 The jdeb Ant task can package up a directory as Debian package. You have to
 provide the control files defining meta information about the package (except
@@ -53,7 +54,9 @@ modifications like prefixing or stripping of paths though.
 
 For more complex permission and ownership adjustments you can use a "ls"
 mapper. It allows you to define permissions and ownerships in a text file and
-even under Windows you will be able to build your debian package.
+even under Windows you will be able to build your debian package. The encoding property 
+does not apply to the "ls" mapper. The mapping files must be encoded as UTF-8.
+
 
 ```xml
     <deb destfile="jdeb.deb" control="${deb}/control">
@@ -64,7 +67,7 @@ even under Windows you will be able to build your debian package.
 ```
 
 The mapper will apply the output of an "ls -laR > mapping.txt" command
-that should look like this
+that should look like this:
 
     ./trunk/target/test-classes/org/vafer/dependency:
     total 176
@@ -75,7 +78,7 @@ that should look like this
     drwxr-xr-x    4 tcurdt  tcurdt   136 Jun 25 03:48 classes
 
 It's also possible to use a `fileset` or even a `tarfileset` to
-specify the set of files to include with their permissions :
+specify the set of files to include with their permissions:
 
 ```xml
     <deb destfile="jdeb.deb" control="${deb}/control">
@@ -106,7 +109,7 @@ uid           | Numerical uid                                          | No; def
 gid           | Numerical gid                                          | No; defaults to 0
 user          | User name                                              | No; defaults to "root"
 group         | User group                                             | No; defaults to "root"
-mode          | Permissions as octet                                   | No; deftauls to 777
+mode          | Permissions as octet                                   | No; defaults to 777
 
 ## Changes file
 

docs/maven.md

@@ -147,6 +147,9 @@ verbose          | Verbose logging
 skip             | Indicates if an execution should be skipped                                                | No; defaults to `false`
 skipSubmodules   | Skip goal on all submodules                                                                | No; defaults to `false`
 skipPOMs         | Skip goal on POM artifacts                                                                 | No; defaults to `true`
+encoding         | The character character encoding to use when reading control and changes files.            | No; defaults to `${project.build.sourceEncoding}`
+
+The encoding property does not apply to the "ls" mapper. The mapping files must be encoded as UTF-8.
 
 If you use the `dataSet` element, you'll need to populate it with a one or
 more `data` elements. A `data` element is used to specify a directory, a

src/main/java/org/vafer/jdeb/ControlBuilder.java

@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
+import java.nio.charset.Charset;
 import java.text.ParseException;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -84,7 +85,7 @@ class ControlBuilder {
      * @throws java.io.IOException
      * @throws java.text.ParseException
      */
-    void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFiles, List<String> conffiles, StringBuilder checksums, File output) throws IOException, ParseException {
+    void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFiles, List<String> conffiles, StringBuilder checksums, File output, Charset encoding) throws IOException, ParseException {
 
         if (packageControlFile == null) {
             throw new FileNotFoundException("No 'control' file found in " + controlFiles.toString());
@@ -123,9 +124,7 @@ void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFil
 
             if (CONFIGURATION_FILENAMES.contains(file.getName()) || MAINTAINER_SCRIPTS.contains(file.getName())) {
 
-                FilteredFile configurationFile = new FilteredFile(new FileInputStream(file), resolver);
-                configurationFile.setOpenToken(openReplaceToken);
-                configurationFile.setCloseToken(closeReplaceToken);
+                FilteredFile configurationFile = new FilteredFile(new FileInputStream(file), resolver, encoding, openReplaceToken, closeReplaceToken);
                 addControlEntry(file.getName(), configurationFile.toString(), outputStream);
 
             } else {
@@ -138,11 +137,11 @@ void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFil
                 // fix line endings for shell scripts
                 InputStream in = new FileInputStream(file);
                 if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) {
-                    byte[] buf = Utils.toUnixLineEndings(in);
+                    byte[] buf = Utils.toUnixLineEndings(in, encoding);
                     in = new ByteArrayInputStream(buf);
                 }
 
-                addControlEntry(file.getName(), IOUtils.toString(in, java.nio.charset.StandardCharsets.UTF_8), outputStream);
+                addControlEntry(file.getName(), IOUtils.toString(in, encoding), outputStream);
 
                 in.close();
             }
@@ -174,6 +173,10 @@ private String createPackageConffilesFile(final List<String> conffiles) {
         return content.toString();
     }
 
+    @Deprecated
+    public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize) throws IOException, ParseException {
+        return createPackageControlFile(file, pDataSize, Charset.defaultCharset(), "[[", "]]");
+    }
 
     /**
      * Creates a package control file from the specified file and adds the
@@ -184,9 +187,10 @@ private String createPackageConffilesFile(final List<String> conffiles) {
      *
      * @param file       the control file
      * @param pDataSize  the size of the installed package
+     * @param encoding   the encoding used to read the files
      */
-    public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize) throws IOException, ParseException {
-        FilteredFile controlFile = new FilteredFile(new FileInputStream(file), resolver);
+    public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize, Charset encoding, String openToken, String closeToken) throws IOException, ParseException {
+        FilteredFile controlFile = new FilteredFile(new FileInputStream(file), resolver, encoding, openToken, closeToken);
         BinaryPackageControlFile packageControlFile = new BinaryPackageControlFile(controlFile.toString());
 
         if (packageControlFile.get("Distribution") == null) {

src/main/java/org/vafer/jdeb/DebMaker.java

@@ -34,6 +34,7 @@
 import org.vafer.jdeb.debian.BinaryPackageControlFile;
 import org.vafer.jdeb.debian.ChangesFile;
 import org.vafer.jdeb.signing.PGPSigner;
+import org.vafer.jdeb.utils.FilteredFile;
 import org.vafer.jdeb.utils.PGPSignatureOutputStream;
 import org.vafer.jdeb.utils.Utils;
 import org.vafer.jdeb.utils.VariableResolver;
@@ -45,6 +46,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -74,6 +76,9 @@ public class DebMaker {
     /** The directory containing the control files to build the package */
     private File control;
 
+    /** The character character encoding to use when reading control and changes files */
+    private Charset encoding = Charset.defaultCharset();
+
     /** The name of the package. Default value if not specified in the control file */
     private String packageName;
 
@@ -140,8 +145,8 @@ public class DebMaker {
     private Long outputTimestampMs;
 
     private VariableResolver variableResolver;
-    private String openReplaceToken;
-    private String closeReplaceToken;
+    private String openReplaceToken = FilteredFile.DEFAULT_OPEN_TOKEN;
+    private String closeReplaceToken = FilteredFile.DEFAULT_CLOSE_TOKEN;
 
     private final Collection<DataProducer> dataProducers = new ArrayList<>();
 
@@ -168,6 +173,18 @@ public void setControl(File control) {
         this.control = control;
     }
 
+    public void setEncoding(Charset encoding) {
+        this.encoding = encoding;
+    }
+
+    public void setEncoding(String encoding) {
+        if (encoding == null || encoding.isEmpty()) {
+            this.encoding = Charset.defaultCharset();
+        } else {
+            this.encoding = Charset.forName(encoding);
+        }
+    }
+
     public void setPackage(String packageName) {
         this.packageName = packageName;
     }
@@ -380,7 +397,7 @@ private void makeChangesFiles(final BinaryPackageControlFile packageControlFile)
 
             if (changesIn != null && changesIn.exists()) {
                 // read the changes form a textfile provider
-                changesProvider = new TextfileChangesProvider(new FileInputStream(changesIn), packageControlFile, outputTimestampMs);
+                changesProvider = new TextfileChangesProvider(new FileInputStream(changesIn), packageControlFile, outputTimestampMs, encoding);
             } else {
                 // create an empty changelog
                 changesProvider = new ChangesProvider() {
@@ -515,7 +532,7 @@ public BinaryPackageControlFile createSignedDeb(Compression compression, final P
 
             console.debug("Building control");
             ControlBuilder controlBuilder = new ControlBuilder(console, variableResolver, openReplaceToken, closeReplaceToken, outputTimestampMs);
-            BinaryPackageControlFile packageControlFile = controlBuilder.createPackageControlFile(new File(control, "control"), size);
+            BinaryPackageControlFile packageControlFile = controlBuilder.createPackageControlFile(new File(control, "control"), size, encoding, openReplaceToken, closeReplaceToken);
             if (packageControlFile.get("Package") == null) {
                 packageControlFile.set("Package", packageName);
             }
@@ -535,7 +552,7 @@ public BinaryPackageControlFile createSignedDeb(Compression compression, final P
                 packageControlFile.set("Homepage", homepage);
             }
 
-            controlBuilder.buildControl(packageControlFile, control.listFiles(), tempConffiles , md5s, tempControl);
+            controlBuilder.buildControl(packageControlFile, control.listFiles(), tempConffiles , md5s, tempControl, encoding);
 
             if (!packageControlFile.isValid()) {
                 throw new PackagingException("Control file fields are invalid " + packageControlFile.invalidFields() +

src/main/java/org/vafer/jdeb/ant/DebAntTask.java

@@ -44,6 +44,9 @@ public class DebAntTask extends MatchingTask {
     /** The directory containing the control files to build the package */
     private File control;
 
+    /** The character character encoding to use when reading control and changes files. */
+    private String encoding;
+
     /** The file containing the PGP keys */
     private File keyring;
 
@@ -92,6 +95,10 @@ public void setControl( File control ) {
         this.control = control;
     }
 
+    public void setEncoding( String encoding ) {
+        this.encoding = encoding;
+    }
+
     public void setChangesIn( File changes ) {
         this.changesIn = changes;
     }
@@ -170,6 +177,7 @@ public void execute() {
         DebMaker debMaker = new DebMaker(console, dataProducers, conffilesProducers);
         debMaker.setDeb(deb);
         debMaker.setControl(control);
+        debMaker.setEncoding(encoding);
         debMaker.setChangesIn(changesIn);
         debMaker.setChangesOut(changesOut);
         debMaker.setChangesSave(changesSave);

src/main/java/org/vafer/jdeb/changes/TextfileChangesProvider.java

@@ -23,6 +23,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
@@ -49,18 +50,28 @@ public final class TextfileChangesProvider implements ChangesProvider {
 
     private final DateFormat fmt;
 
+    @Deprecated
     public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile ) throws IOException, ParseException {
-        this(pInput, packageControlFile, null);
+        this(pInput, packageControlFile, null, Charset.defaultCharset());
     }
 
+    @Deprecated
     public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Long outputTimestampMs ) throws IOException, ParseException {
+        this(pInput, packageControlFile, outputTimestampMs, Charset.defaultCharset());
+    }
+
+    public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Charset encoding ) throws IOException, ParseException {
+        this(pInput, packageControlFile, null, encoding);
+    }
+
+    public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Long outputTimestampMs, final Charset encoding ) throws IOException, ParseException {
 
         fmt = new SimpleDateFormat("HH:mm dd.MM.yyyy");
         if (outputTimestampMs != null) {
             fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
         }
 
-        final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput));
+        final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput, encoding));
 
         String packageName = packageControlFile.get("Package");
         String version = packageControlFile.get("Version");

src/main/java/org/vafer/jdeb/mapping/LsMapper.java

@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -178,7 +179,7 @@ private TarArchiveEntry readFile( final BufferedReader reader, final String base
     private Map<String, TarArchiveEntry> parse( final InputStream pInput ) throws IOException, ParseError {
         final Map<String, TarArchiveEntry> mapping = new HashMap<>();
 
-        final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput));
+        final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput, StandardCharsets.UTF_8));
 
         boolean first = true;
         while (true) {

src/main/java/org/vafer/jdeb/maven/DebMojo.java

@@ -78,6 +78,14 @@ public class DebMojo extends AbstractMojo {
     @Parameter
     private String name;
 
+    /**
+     * The character character encoding to use when reading control and changes files.
+     * 
+     * @since 1.13
+     */
+    @Parameter(defaultValue = "${project.build.sourceEncoding}")
+    private String encoding;
+
     /**
      * Defines the pattern of the name of final artifacts. Possible
      * substitutions are [[baseDir]] [[buildDir]] [[artifactId]] [[version]]
@@ -587,6 +595,7 @@ public void produce( final DataConsumer receiver ) {
             DebMaker debMaker = new DebMaker(console, dataProducers, conffileProducers);
             debMaker.setDeb(debFile);
             debMaker.setControl(controlDirFile);
+            debMaker.setEncoding(encoding);
             debMaker.setPackage(getProject().getArtifactId());
             debMaker.setDescription(getProject().getDescription());
             debMaker.setHomepage(getProject().getUrl());

src/main/java/org/vafer/jdeb/utils/FilteredFile.java

@@ -20,29 +20,46 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
 public class FilteredFile {
 
-    private String openToken = "[[";
-    private String closeToken = "]]";
+    public static final String DEFAULT_OPEN_TOKEN = "[[";
+    public static final String DEFAULT_CLOSE_TOKEN = "]]";
+
+    private String openToken;
+    private String closeToken;
     private List<String> lines = new ArrayList<>();
 
+    @Deprecated
     public FilteredFile(InputStream in, VariableResolver resolver) throws IOException {
-        parse(in, resolver);
+        this(in, resolver, Charset.defaultCharset());
+    }
+
+    public FilteredFile(InputStream in, VariableResolver resolver, Charset encoding) throws IOException {
+        this(in, resolver, encoding, DEFAULT_OPEN_TOKEN, DEFAULT_CLOSE_TOKEN);
+    }
+
+    public FilteredFile(InputStream in, VariableResolver resolver, Charset encoding, String openToken, String closeToken) throws IOException {
+        this.openToken = openToken;
+        this.closeToken = closeToken;
+        parse(in, resolver, encoding);
     }
 
+    @Deprecated
     public void setOpenToken(String token) {
         openToken = token;
     }
 
+    @Deprecated
     public void setCloseToken(String token) {
         closeToken = token;
     }
 
-    private void parse(InputStream in, VariableResolver resolver) throws IOException {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+    private void parse(InputStream in, VariableResolver resolver, Charset encoding) throws IOException {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding))) {
             String line;
             while ((line = reader.readLine()) != null) {
                 if (resolver != null) {