diff --git a/docs/ant.md b/docs/ant.md index a0e81cdf5..79e7566c4 100644 --- a/docs/ant.md +++ b/docs/ant.md @@ -1,18 +1,19 @@ # How to use jdeb with Ant -Attribute | Description | Required --------------- | ---------------------------------------------------------------------------- | -------------------------- -destfile | The debian package to be generated | Yes -control | The directory containing the control files | Yes -compression | Compression method for the data file (`gzip`, `bzip2`, `xz` or `none`) | No; defaults to `gzip` -verbose | Print detailed info during the package generation | No; defaults to `false` -keyring | The file containing the PGP keys | No -key | The name of the key to be used in the keyring | No -passphrase | The passphrase to use the key | No -changesIn | The changes to add | No -changesOut | The changes file generated | No -changesSave | The merged changes file | No -changesEnabled | Enable the creation of the changes file | No; defaults to `true` +Attribute | Description | Required +-------------- | ------------------------------------------------------------------------------ | -------------------------- +destfile | The debian package to be generated | Yes +control | The directory containing the control files | Yes +compression | Compression method for the data file (`gzip`, `bzip2`, `xz` or `none`) | No; defaults to `gzip` +verbose | Print detailed info during the package generation | No; defaults to `false` +keyring | The file containing the PGP keys | No +key | The name of the key to be used in the keyring | No +passphrase | The passphrase to use the key | No +changesIn | The changes to add | No +changesOut | The changes file generated | No +changesSave | The merged changes file | No +changesEnabled | Enable the creation of the changes file | No; defaults to `true` +encoding | The character character encoding to use when reading control and changes files | No; defaults to default charset of the JVM The jdeb Ant task can package up a directory as Debian package. You have to provide the control files defining meta information about the package (except @@ -53,7 +54,9 @@ modifications like prefixing or stripping of paths though. For more complex permission and ownership adjustments you can use a "ls" mapper. It allows you to define permissions and ownerships in a text file and -even under Windows you will be able to build your debian package. +even under Windows you will be able to build your debian package. The encoding property +does not apply to the "ls" mapper. The mapping files must be encoded as UTF-8. + ```xml @@ -64,7 +67,7 @@ even under Windows you will be able to build your debian package. ``` The mapper will apply the output of an "ls -laR > mapping.txt" command -that should look like this +that should look like this: ./trunk/target/test-classes/org/vafer/dependency: total 176 @@ -75,7 +78,7 @@ that should look like this drwxr-xr-x 4 tcurdt tcurdt 136 Jun 25 03:48 classes It's also possible to use a `fileset` or even a `tarfileset` to -specify the set of files to include with their permissions : +specify the set of files to include with their permissions: ```xml @@ -106,7 +109,7 @@ uid | Numerical uid | No; def gid | Numerical gid | No; defaults to 0 user | User name | No; defaults to "root" group | User group | No; defaults to "root" -mode | Permissions as octet | No; deftauls to 777 +mode | Permissions as octet | No; defaults to 777 ## Changes file diff --git a/docs/maven.md b/docs/maven.md index 2ae16a89a..e12a4b464 100644 --- a/docs/maven.md +++ b/docs/maven.md @@ -147,6 +147,9 @@ verbose | Verbose logging skip | Indicates if an execution should be skipped | No; defaults to `false` skipSubmodules | Skip goal on all submodules | No; defaults to `false` skipPOMs | Skip goal on POM artifacts | No; defaults to `true` +encoding | The character character encoding to use when reading control and changes files. | No; defaults to `${project.build.sourceEncoding}` + +The encoding property does not apply to the "ls" mapper. The mapping files must be encoded as UTF-8. If you use the `dataSet` element, you'll need to populate it with a one or more `data` elements. A `data` element is used to specify a directory, a diff --git a/src/main/java/org/vafer/jdeb/ControlBuilder.java b/src/main/java/org/vafer/jdeb/ControlBuilder.java index 1c6484c94..297950dca 100644 --- a/src/main/java/org/vafer/jdeb/ControlBuilder.java +++ b/src/main/java/org/vafer/jdeb/ControlBuilder.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.nio.charset.Charset; import java.text.ParseException; import java.util.Arrays; import java.util.Comparator; @@ -84,7 +85,7 @@ class ControlBuilder { * @throws java.io.IOException * @throws java.text.ParseException */ - void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFiles, List conffiles, StringBuilder checksums, File output) throws IOException, ParseException { + void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFiles, List conffiles, StringBuilder checksums, File output, Charset encoding) throws IOException, ParseException { if (packageControlFile == null) { throw new FileNotFoundException("No 'control' file found in " + controlFiles.toString()); @@ -123,9 +124,7 @@ void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFil if (CONFIGURATION_FILENAMES.contains(file.getName()) || MAINTAINER_SCRIPTS.contains(file.getName())) { - FilteredFile configurationFile = new FilteredFile(new FileInputStream(file), resolver); - configurationFile.setOpenToken(openReplaceToken); - configurationFile.setCloseToken(closeReplaceToken); + FilteredFile configurationFile = new FilteredFile(new FileInputStream(file), resolver, encoding, openReplaceToken, closeReplaceToken); addControlEntry(file.getName(), configurationFile.toString(), outputStream); } else { @@ -138,11 +137,11 @@ void buildControl(BinaryPackageControlFile packageControlFile, File[] controlFil // fix line endings for shell scripts InputStream in = new FileInputStream(file); if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) { - byte[] buf = Utils.toUnixLineEndings(in); + byte[] buf = Utils.toUnixLineEndings(in, encoding); in = new ByteArrayInputStream(buf); } - addControlEntry(file.getName(), IOUtils.toString(in, java.nio.charset.StandardCharsets.UTF_8), outputStream); + addControlEntry(file.getName(), IOUtils.toString(in, encoding), outputStream); in.close(); } @@ -174,6 +173,10 @@ private String createPackageConffilesFile(final List conffiles) { return content.toString(); } + @Deprecated + public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize) throws IOException, ParseException { + return createPackageControlFile(file, pDataSize, Charset.defaultCharset(), "[[", "]]"); + } /** * Creates a package control file from the specified file and adds the @@ -184,9 +187,10 @@ private String createPackageConffilesFile(final List conffiles) { * * @param file the control file * @param pDataSize the size of the installed package + * @param encoding the encoding used to read the files */ - public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize) throws IOException, ParseException { - FilteredFile controlFile = new FilteredFile(new FileInputStream(file), resolver); + public BinaryPackageControlFile createPackageControlFile(File file, BigInteger pDataSize, Charset encoding, String openToken, String closeToken) throws IOException, ParseException { + FilteredFile controlFile = new FilteredFile(new FileInputStream(file), resolver, encoding, openToken, closeToken); BinaryPackageControlFile packageControlFile = new BinaryPackageControlFile(controlFile.toString()); if (packageControlFile.get("Distribution") == null) { diff --git a/src/main/java/org/vafer/jdeb/DebMaker.java b/src/main/java/org/vafer/jdeb/DebMaker.java index a21908090..02b9d88bd 100644 --- a/src/main/java/org/vafer/jdeb/DebMaker.java +++ b/src/main/java/org/vafer/jdeb/DebMaker.java @@ -34,6 +34,7 @@ import org.vafer.jdeb.debian.BinaryPackageControlFile; import org.vafer.jdeb.debian.ChangesFile; import org.vafer.jdeb.signing.PGPSigner; +import org.vafer.jdeb.utils.FilteredFile; import org.vafer.jdeb.utils.PGPSignatureOutputStream; import org.vafer.jdeb.utils.Utils; import org.vafer.jdeb.utils.VariableResolver; @@ -45,6 +46,7 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -74,6 +76,9 @@ public class DebMaker { /** The directory containing the control files to build the package */ private File control; + /** The character character encoding to use when reading control and changes files */ + private Charset encoding = Charset.defaultCharset(); + /** The name of the package. Default value if not specified in the control file */ private String packageName; @@ -140,8 +145,8 @@ public class DebMaker { private Long outputTimestampMs; private VariableResolver variableResolver; - private String openReplaceToken; - private String closeReplaceToken; + private String openReplaceToken = FilteredFile.DEFAULT_OPEN_TOKEN; + private String closeReplaceToken = FilteredFile.DEFAULT_CLOSE_TOKEN; private final Collection dataProducers = new ArrayList<>(); @@ -168,6 +173,18 @@ public void setControl(File control) { this.control = control; } + public void setEncoding(Charset encoding) { + this.encoding = encoding; + } + + public void setEncoding(String encoding) { + if (encoding == null || encoding.isEmpty()) { + this.encoding = Charset.defaultCharset(); + } else { + this.encoding = Charset.forName(encoding); + } + } + public void setPackage(String packageName) { this.packageName = packageName; } @@ -380,7 +397,7 @@ private void makeChangesFiles(final BinaryPackageControlFile packageControlFile) if (changesIn != null && changesIn.exists()) { // read the changes form a textfile provider - changesProvider = new TextfileChangesProvider(new FileInputStream(changesIn), packageControlFile, outputTimestampMs); + changesProvider = new TextfileChangesProvider(new FileInputStream(changesIn), packageControlFile, outputTimestampMs, encoding); } else { // create an empty changelog changesProvider = new ChangesProvider() { @@ -515,7 +532,7 @@ public BinaryPackageControlFile createSignedDeb(Compression compression, final P console.debug("Building control"); ControlBuilder controlBuilder = new ControlBuilder(console, variableResolver, openReplaceToken, closeReplaceToken, outputTimestampMs); - BinaryPackageControlFile packageControlFile = controlBuilder.createPackageControlFile(new File(control, "control"), size); + BinaryPackageControlFile packageControlFile = controlBuilder.createPackageControlFile(new File(control, "control"), size, encoding, openReplaceToken, closeReplaceToken); if (packageControlFile.get("Package") == null) { packageControlFile.set("Package", packageName); } @@ -535,7 +552,7 @@ public BinaryPackageControlFile createSignedDeb(Compression compression, final P packageControlFile.set("Homepage", homepage); } - controlBuilder.buildControl(packageControlFile, control.listFiles(), tempConffiles , md5s, tempControl); + controlBuilder.buildControl(packageControlFile, control.listFiles(), tempConffiles , md5s, tempControl, encoding); if (!packageControlFile.isValid()) { throw new PackagingException("Control file fields are invalid " + packageControlFile.invalidFields() + diff --git a/src/main/java/org/vafer/jdeb/ant/DebAntTask.java b/src/main/java/org/vafer/jdeb/ant/DebAntTask.java index 968c9257f..7cc8b7ad4 100644 --- a/src/main/java/org/vafer/jdeb/ant/DebAntTask.java +++ b/src/main/java/org/vafer/jdeb/ant/DebAntTask.java @@ -44,6 +44,9 @@ public class DebAntTask extends MatchingTask { /** The directory containing the control files to build the package */ private File control; + /** The character character encoding to use when reading control and changes files. */ + private String encoding; + /** The file containing the PGP keys */ private File keyring; @@ -92,6 +95,10 @@ public void setControl( File control ) { this.control = control; } + public void setEncoding( String encoding ) { + this.encoding = encoding; + } + public void setChangesIn( File changes ) { this.changesIn = changes; } @@ -170,6 +177,7 @@ public void execute() { DebMaker debMaker = new DebMaker(console, dataProducers, conffilesProducers); debMaker.setDeb(deb); debMaker.setControl(control); + debMaker.setEncoding(encoding); debMaker.setChangesIn(changesIn); debMaker.setChangesOut(changesOut); debMaker.setChangesSave(changesSave); diff --git a/src/main/java/org/vafer/jdeb/changes/TextfileChangesProvider.java b/src/main/java/org/vafer/jdeb/changes/TextfileChangesProvider.java index c97ddd59a..826088d6e 100644 --- a/src/main/java/org/vafer/jdeb/changes/TextfileChangesProvider.java +++ b/src/main/java/org/vafer/jdeb/changes/TextfileChangesProvider.java @@ -23,6 +23,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.nio.charset.Charset; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -49,18 +50,28 @@ public final class TextfileChangesProvider implements ChangesProvider { private final DateFormat fmt; + @Deprecated public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile ) throws IOException, ParseException { - this(pInput, packageControlFile, null); + this(pInput, packageControlFile, null, Charset.defaultCharset()); } + @Deprecated public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Long outputTimestampMs ) throws IOException, ParseException { + this(pInput, packageControlFile, outputTimestampMs, Charset.defaultCharset()); + } + + public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Charset encoding ) throws IOException, ParseException { + this(pInput, packageControlFile, null, encoding); + } + + public TextfileChangesProvider( final InputStream pInput, final BinaryPackageControlFile packageControlFile, final Long outputTimestampMs, final Charset encoding ) throws IOException, ParseException { fmt = new SimpleDateFormat("HH:mm dd.MM.yyyy"); if (outputTimestampMs != null) { fmt.setTimeZone(TimeZone.getTimeZone("UTC")); } - final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput)); + final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput, encoding)); String packageName = packageControlFile.get("Package"); String version = packageControlFile.get("Version"); diff --git a/src/main/java/org/vafer/jdeb/mapping/LsMapper.java b/src/main/java/org/vafer/jdeb/mapping/LsMapper.java index 2d587a189..4defc3b9d 100644 --- a/src/main/java/org/vafer/jdeb/mapping/LsMapper.java +++ b/src/main/java/org/vafer/jdeb/mapping/LsMapper.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -178,7 +179,7 @@ private TarArchiveEntry readFile( final BufferedReader reader, final String base private Map parse( final InputStream pInput ) throws IOException, ParseError { final Map mapping = new HashMap<>(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput)); + final BufferedReader reader = new BufferedReader(new InputStreamReader(pInput, StandardCharsets.UTF_8)); boolean first = true; while (true) { diff --git a/src/main/java/org/vafer/jdeb/maven/DebMojo.java b/src/main/java/org/vafer/jdeb/maven/DebMojo.java index b0cbb9adf..034dea37b 100644 --- a/src/main/java/org/vafer/jdeb/maven/DebMojo.java +++ b/src/main/java/org/vafer/jdeb/maven/DebMojo.java @@ -78,6 +78,14 @@ public class DebMojo extends AbstractMojo { @Parameter private String name; + /** + * The character character encoding to use when reading control and changes files. + * + * @since 1.13 + */ + @Parameter(defaultValue = "${project.build.sourceEncoding}") + private String encoding; + /** * Defines the pattern of the name of final artifacts. Possible * substitutions are [[baseDir]] [[buildDir]] [[artifactId]] [[version]] @@ -587,6 +595,7 @@ public void produce( final DataConsumer receiver ) { DebMaker debMaker = new DebMaker(console, dataProducers, conffileProducers); debMaker.setDeb(debFile); debMaker.setControl(controlDirFile); + debMaker.setEncoding(encoding); debMaker.setPackage(getProject().getArtifactId()); debMaker.setDescription(getProject().getDescription()); debMaker.setHomepage(getProject().getUrl()); diff --git a/src/main/java/org/vafer/jdeb/utils/FilteredFile.java b/src/main/java/org/vafer/jdeb/utils/FilteredFile.java index 274d37545..a3fbfff0e 100644 --- a/src/main/java/org/vafer/jdeb/utils/FilteredFile.java +++ b/src/main/java/org/vafer/jdeb/utils/FilteredFile.java @@ -20,29 +20,46 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; public class FilteredFile { - private String openToken = "[["; - private String closeToken = "]]"; + public static final String DEFAULT_OPEN_TOKEN = "[["; + public static final String DEFAULT_CLOSE_TOKEN = "]]"; + + private String openToken; + private String closeToken; private List lines = new ArrayList<>(); + @Deprecated public FilteredFile(InputStream in, VariableResolver resolver) throws IOException { - parse(in, resolver); + this(in, resolver, Charset.defaultCharset()); + } + + public FilteredFile(InputStream in, VariableResolver resolver, Charset encoding) throws IOException { + this(in, resolver, encoding, DEFAULT_OPEN_TOKEN, DEFAULT_CLOSE_TOKEN); + } + + public FilteredFile(InputStream in, VariableResolver resolver, Charset encoding, String openToken, String closeToken) throws IOException { + this.openToken = openToken; + this.closeToken = closeToken; + parse(in, resolver, encoding); } + @Deprecated public void setOpenToken(String token) { openToken = token; } + @Deprecated public void setCloseToken(String token) { closeToken = token; } - private void parse(InputStream in, VariableResolver resolver) throws IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + private void parse(InputStream in, VariableResolver resolver, Charset encoding) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding))) { String line; while ((line = reader.readLine()) != null) { if (resolver != null) { diff --git a/src/main/java/org/vafer/jdeb/utils/Utils.java b/src/main/java/org/vafer/jdeb/utils/Utils.java index e77aeaabf..c84338f80 100644 --- a/src/main/java/org/vafer/jdeb/utils/Utils.java +++ b/src/main/java/org/vafer/jdeb/utils/Utils.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; @@ -210,13 +211,18 @@ public static String replaceVariables( final VariableResolver pResolver, final S return out.toString(); } + @Deprecated + public static byte[] toUnixLineEndings( InputStream input ) throws IOException { + String encoding = "ISO-8859-1"; + return toUnixLineEndings(input, Charset.forName(encoding)); + } + /** * Replaces new line delimiters in the input stream with the Unix line feed. * * @param input */ - public static byte[] toUnixLineEndings( InputStream input ) throws IOException { - String encoding = "ISO-8859-1"; + public static byte[] toUnixLineEndings( InputStream input, Charset encoding ) throws IOException { FixCrLfFilter filter = new FixCrLfFilter(new InputStreamReader(input, encoding)); filter.setEol(FixCrLfFilter.CrLf.newInstance("unix")); diff --git a/src/test/java/org/vafer/jdeb/DebMakerLongNameTestCase.java b/src/test/java/org/vafer/jdeb/DebMakerLongNameTestCase.java index 8291e0e4a..bf102e5b4 100644 --- a/src/test/java/org/vafer/jdeb/DebMakerLongNameTestCase.java +++ b/src/test/java/org/vafer/jdeb/DebMakerLongNameTestCase.java @@ -16,6 +16,7 @@ import org.vafer.jdeb.producers.DataProducerPathTemplate; import java.io.File; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -69,6 +70,7 @@ public void testLongLinkName() throws Exception { List producers = Arrays.asList(linkProducer, dataProducer); DebMaker maker = new DebMaker(new NullConsole(), producers, null); + maker.setEncoding(StandardCharsets.UTF_8); maker.setControl(new File(getClass().getResource("deb/control").toURI())); maker.setTarLongFileMode(tarLongFileMode); maker.setDeb(deb); diff --git a/src/test/java/org/vafer/jdeb/DebMakerTestCase.java b/src/test/java/org/vafer/jdeb/DebMakerTestCase.java index 3846ed782..61ae8fc18 100644 --- a/src/test/java/org/vafer/jdeb/DebMakerTestCase.java +++ b/src/test/java/org/vafer/jdeb/DebMakerTestCase.java @@ -37,6 +37,7 @@ import org.junit.Assert; import org.junit.Test; import org.vafer.jdeb.debian.BinaryPackageControlFile; +import org.vafer.jdeb.debian.ControlFile; import org.vafer.jdeb.maven.Data; import org.vafer.jdeb.maven.Mapper; import org.vafer.jdeb.producers.DataProducerArchive; @@ -76,6 +77,7 @@ public void testCreation() throws Exception { DebMaker maker = new DebMaker(new NullConsole(), Arrays.asList(data), Arrays.asList(conffile1, conffile2)); + maker.setEncoding(StandardCharsets.UTF_8); maker.setControl(new File(getClass().getResource("deb/control").toURI())); maker.setDeb(deb); @@ -118,6 +120,213 @@ public void visit(TarArchiveEntry entry, byte[] content) throws IOException { assertTrue("Cannot delete the file " + deb, deb.delete()); } + @Test + public void testCreationEncoding() throws Exception { + DataProducer[] data = prepareData(); + File deb = File.createTempFile("jdeb", ".deb"); + + File conffile = new File(getClass().getResource("deb/data.tgz").toURI()); + + Data conffile1 = new Data(); + conffile1.setType("file"); + conffile1.setSrc(conffile); + conffile1.setDst("/absolute/path/to/configuration"); + conffile1.setConffile(true); + Data conffile2 = new Data(); + conffile2.setType("file"); + conffile2.setSrc(conffile); + conffile2.setConffile(true); + + Mapper mapper = new Mapper(); + FieldUtils.writeField(mapper, "type", "perm", true); + FieldUtils.writeField(mapper, "prefix", "/absolute/prefix", true); + FieldUtils.writeField(conffile2, "mapper", mapper, true); + + DebMaker maker = new DebMaker(new NullConsole(), Arrays.asList(data), Arrays.asList(conffile1, conffile2)); + + maker.setControl(new File(getClass().getResource("deb/controlencoding").toURI())); + maker.setDeb(deb); + + // first step: invalid encoding + maker.setEncoding(StandardCharsets.ISO_8859_1); + + BinaryPackageControlFile packageControlFile = maker.createDeb(Compression.GZIP); + + assertTrue(packageControlFile.isValid()); + + ArchiveWalker.walkControl(deb, new ArchiveVisitor() { + @Override + public void visit(TarArchiveEntry entry, byte[] content) throws IOException { + if (entry.getName().equals("./control")) { + try { + ControlFile controlFile = new BinaryPackageControlFile(org.apache.commons.io.IOUtils.toString(new ByteArrayInputStream(content), StandardCharsets.UTF_8)); + assertNotEquals("the encoding is valid but should be wrong", controlFile.get("Maintainer"), "ジョン Doe "); + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().equals("./postinst") || entry.getName().equals("./prerm")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("# P")) { + assertFalse("the encoding is valid but should be wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().startsWith("./shell_")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("# Custom script")) { + assertFalse("the encoding is valid but should be wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().equals("./text.txt")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("Text file")) { + assertFalse("the encoding is valid but should be wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + } + }); + + // second step: valid encoding + maker.setEncoding(StandardCharsets.UTF_8); + + packageControlFile = maker.createDeb(Compression.GZIP); + + assertTrue(packageControlFile.isValid()); + + ArchiveWalker.walkControl(deb, new ArchiveVisitor() { + @Override + public void visit(TarArchiveEntry entry, byte[] content) throws IOException { + if (entry.getName().equals("./control")) { + try { + ControlFile controlFile = new BinaryPackageControlFile(org.apache.commons.io.IOUtils.toString(new ByteArrayInputStream(content), StandardCharsets.UTF_8)); + assertEquals("the encoding is wrong", "ジョン Doe ", controlFile.get("Maintainer")); + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().equals("./postinst") || entry.getName().equals("./prerm")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("# P")) { + assertTrue("the encoding is wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().startsWith("./shell_")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("# Custom script")) { + assertTrue("the encoding is valid but should be wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().equals("./text.txt")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("Text file")) { + assertTrue("the encoding is valid but should be wrong", line.endsWith("created by ジョン")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + } + }); + } + + @Test + public void testCreationCustomToken() throws Exception { + DataProducer[] data = prepareData(); + File deb = File.createTempFile("jdeb", ".deb"); + + File conffile = new File(getClass().getResource("deb/data.tgz").toURI()); + + Map map = new HashMap<>(); + map.put("name", "actualName"); + map.put("version", "actualVersion"); + map.put("maintainer", "John Doe "); + MapVariableResolver variableResolver = new MapVariableResolver(map); + + Data conffile1 = new Data(); + conffile1.setType("file"); + conffile1.setSrc(conffile); + conffile1.setDst("/absolute/path/to/configuration"); + conffile1.setConffile(true); + Data conffile2 = new Data(); + conffile2.setType("file"); + conffile2.setSrc(conffile); + conffile2.setConffile(true); + + Mapper mapper = new Mapper(); + FieldUtils.writeField(mapper, "type", "perm", true); + FieldUtils.writeField(mapper, "prefix", "/absolute/prefix", true); + FieldUtils.writeField(conffile2, "mapper", mapper, true); + + DebMaker maker = + new DebMaker(new NullConsole(), Arrays.asList(data), Arrays.asList(conffile1, conffile2)); + maker.setEncoding(StandardCharsets.UTF_8); + maker.setControl(new File(getClass().getResource("deb/controlcustomtoken").toURI())); + maker.setDeb(deb); + maker.setResolver(variableResolver); + maker.setOpenReplaceToken("{[{"); + maker.setCloseReplaceToken("}]}"); + + BinaryPackageControlFile packageControlFile = maker.createDeb(Compression.GZIP); + + assertTrue(packageControlFile.isValid()); + + final Map filesInDeb = new HashMap<>(); + + final Set actualConffileContent = new HashSet<>(); + + ArchiveWalker.walkControl(deb, new ArchiveVisitor() { + @Override + public void visit(TarArchiveEntry entry, byte[] content) throws IOException { + if (entry.getName().equals("./control")) { + try { + ControlFile controlFile = new BinaryPackageControlFile(org.apache.commons.io.IOUtils.toString(new ByteArrayInputStream(content), StandardCharsets.UTF_8)); + assertEquals("variable substitution failed", "John Doe ", controlFile.get("Maintainer")); + } catch(Exception e) { + throw new IOException(e); + } + } + else if (entry.getName().equals("./postinst") || entry.getName().equals("./prerm")) { + try { + for(String line : org.apache.commons.io.IOUtils.readLines(new ByteArrayInputStream(content), StandardCharsets.UTF_8)) { + if(line.startsWith("# P")) { + assertTrue("variable substitution failed", line.endsWith("actualName actualVersion")); + } + } + } catch(Exception e) { + throw new IOException(e); + } + } + } + }); + } + @Test public void testControlFilesPermissions() throws Exception { File deb = new File("target/test-classes/test-control.deb"); @@ -128,6 +337,7 @@ public void testControlFilesPermissions() throws Exception { Collection producers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); Collection conffileProducers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); DebMaker maker = new DebMaker(new NullConsole(), producers, conffileProducers); + maker.setEncoding(StandardCharsets.UTF_8); maker.setDeb(deb); maker.setControl(new File("target/test-classes/org/vafer/jdeb/deb/control")); @@ -174,6 +384,7 @@ public void testControlFilesVariables() throws Exception { Collection producers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); Collection conffileProducers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); DebMaker maker = new DebMaker(new NullConsole(), producers, conffileProducers); + maker.setEncoding(StandardCharsets.UTF_8); maker.setDeb(deb); maker.setControl(new File("target/test-classes/org/vafer/jdeb/deb/control")); maker.setResolver(new MapVariableResolver(variables)); @@ -210,6 +421,7 @@ public void testDependsIsOmittedWhenEmpty() throws Exception { Collection producers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); Collection conffileProducers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); DebMaker maker = new DebMaker(new NullConsole(), producers, conffileProducers); + maker.setEncoding(StandardCharsets.UTF_8); maker.setDeb(deb); maker.setControl(new File("target/test-classes/org/vafer/jdeb/deb/controlwithoutdepends")); @@ -241,6 +453,7 @@ public void testDependsIsIncludedIfSet() throws Exception { Collection producers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); Collection conffileProducers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); DebMaker maker = new DebMaker(new NullConsole(), producers, conffileProducers); + maker.setEncoding(StandardCharsets.UTF_8); maker.setDeb(deb); maker.setControl(new File("target/test-classes/org/vafer/jdeb/deb/controlwithoutdepends")); @@ -274,6 +487,7 @@ public void testConstantModifiedTime() throws Exception { Collection confFileProducers = Arrays.asList(new DataProducer[] {new EmptyDataProducer()}); DebMaker maker = new DebMaker(new NullConsole(), Arrays.asList(data), confFileProducers); + maker.setEncoding(StandardCharsets.UTF_8); maker.setControl(new File(getClass().getResource("deb/control").toURI())); maker.setDeb(deb); maker.setOutputTimestampMs(EXPECTED_MODIFIED_TIME); @@ -292,6 +506,7 @@ public void testConstantModifiedTime() throws Exception { public void testErrorPropagation() throws Exception { File deb = File.createTempFile("jdeb", ".deb"); DebMaker maker = new DebMaker(new NullConsole(), List.of(new UseNullAsInputStream()), null); + maker.setEncoding(StandardCharsets.UTF_8); maker.setControl(new File(getClass().getResource("deb/control").toURI())); maker.setDeb(deb); @@ -312,6 +527,7 @@ public void testChangesValidation() throws Exception { File directory = new File(getClass().getResource("deb/data").toURI()); DebMaker maker = new DebMaker(new NullConsole(), List.of(new UseNullAsInputStream()), null); + maker.setEncoding(StandardCharsets.UTF_8); assertThrows(PackagingException.class, maker::validate); maker.setControl(new File(getClass().getResource("deb/control").toURI())); diff --git a/src/test/java/org/vafer/jdeb/changes/ChangesFileBuilderTestCase.java b/src/test/java/org/vafer/jdeb/changes/ChangesFileBuilderTestCase.java index bf7c5de4a..f11372baa 100644 --- a/src/test/java/org/vafer/jdeb/changes/ChangesFileBuilderTestCase.java +++ b/src/test/java/org/vafer/jdeb/changes/ChangesFileBuilderTestCase.java @@ -16,6 +16,7 @@ package org.vafer.jdeb.changes; import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.TimeZone; import static java.nio.charset.StandardCharsets.*; @@ -55,7 +56,7 @@ public void testChangedByNotSet() throws Exception { packageControlFile.set("Version", "version"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); @@ -85,7 +86,7 @@ public void testChangedByFromControl() throws Exception { packageControlFile.set("Maintainer", "tcurdt@joost.com"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); @@ -115,7 +116,7 @@ public void testChangedByFromChangesProvider() throws Exception { packageControlFile.set("Maintainer", "tcurdt@joost.com"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); @@ -144,7 +145,7 @@ public void testReproducible() throws Exception { packageControlFile.set("Version", "version"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, 1175385600000L); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, 1175385600000L, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); diff --git a/src/test/java/org/vafer/jdeb/changes/TextfileChangesProviderTestCase.java b/src/test/java/org/vafer/jdeb/changes/TextfileChangesProviderTestCase.java index 0da392531..212a0e263 100644 --- a/src/test/java/org/vafer/jdeb/changes/TextfileChangesProviderTestCase.java +++ b/src/test/java/org/vafer/jdeb/changes/TextfileChangesProviderTestCase.java @@ -16,6 +16,7 @@ package org.vafer.jdeb.changes; import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.TimeZone; import org.junit.After; @@ -61,7 +62,7 @@ public void testParsing() throws Exception { packageControlFile.set("Distribution", "distribution"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); @@ -90,7 +91,7 @@ public void testDistributionFromChangesProvider() throws Exception { packageControlFile.set("Version", "version"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); @@ -123,7 +124,7 @@ public void testReproducible() throws Exception { packageControlFile.set("Distribution", "distribution"); packageControlFile.set("Date", "Mon, 20 Aug 2007 15:25:57 +0200"); - final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, 1175385600000L); + final TextfileChangesProvider provider = new TextfileChangesProvider(new ByteArrayInputStream(input.getBytes(UTF_8)), packageControlFile, 1175385600000L, StandardCharsets.UTF_8); final ChangeSet[] changeSets = provider.getChangesSets(); assertNotNull(changeSets); diff --git a/src/test/java/org/vafer/jdeb/signing/DebMakerTestCase.java b/src/test/java/org/vafer/jdeb/signing/DebMakerTestCase.java index 0d5da0bbb..e3abc4003 100644 --- a/src/test/java/org/vafer/jdeb/signing/DebMakerTestCase.java +++ b/src/test/java/org/vafer/jdeb/signing/DebMakerTestCase.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -73,6 +74,7 @@ public void testCreation() throws Exception { File deb = File.createTempFile("jdeb", ".deb"); DebMaker maker = new DebMaker(new NullConsole(), Arrays.asList(data), null); + maker.setEncoding(StandardCharsets.UTF_8); maker.setControl(new File(getClass().getResource("../deb/control").toURI())); maker.setDeb(deb); diff --git a/src/test/java/org/vafer/jdeb/utils/FilteredFileTestCase.java b/src/test/java/org/vafer/jdeb/utils/FilteredFileTestCase.java index 44bcec80d..364c7fb14 100644 --- a/src/test/java/org/vafer/jdeb/utils/FilteredFileTestCase.java +++ b/src/test/java/org/vafer/jdeb/utils/FilteredFileTestCase.java @@ -18,6 +18,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -45,7 +46,17 @@ public void setUp() throws Exception { public void testTokenSubstitution() throws Exception { InputStream in = new ReaderInputStream(new StringReader("#!/bin/sh\ncat [[artifactId]][[myProperty1]] \necho '[[myProperty2]]'\n")); - FilteredFile placeHolder = new FilteredFile(in, variableResolver); + FilteredFile placeHolder = new FilteredFile(in, variableResolver, StandardCharsets.UTF_8); + + String actual = placeHolder.toString(); + assertEquals("#!/bin/sh\ncat jdebcustom1 \necho 'custom2'\n", actual); + } + + @Test + public void testCustomTokenSubstitution() throws Exception { + InputStream in = new ReaderInputStream(new StringReader("#!/bin/sh\ncat {[{artifactId}]}{[{myProperty1}]} \necho '{[{myProperty2}]}'\n")); + + FilteredFile placeHolder = new FilteredFile(in, variableResolver, StandardCharsets.UTF_8, "{[{", "}]}"); String actual = placeHolder.toString(); assertEquals("#!/bin/sh\ncat jdebcustom1 \necho 'custom2'\n", actual); @@ -65,7 +76,7 @@ public void testTokenSubstitutionWithinOpenCloseTokens() throws Exception { String expected = pair.getValue(); InputStream in = new ReaderInputStream(new StringReader(input)); - FilteredFile placeHolder = new FilteredFile(in, variableResolver); + FilteredFile placeHolder = new FilteredFile(in, variableResolver, StandardCharsets.UTF_8); String actual = placeHolder.toString(); assertEquals("unexpected resolve for " + input, expected, actual); @@ -86,7 +97,7 @@ public void testVariableSubstitution() throws Exception { + "NoResolve1: test[[test\n" + "NoResolve2: [[test]]\n"; - FilteredFile filteredFile = new FilteredFile(new ByteArrayInputStream(controlFile.getBytes()), new MapVariableResolver(map)); + FilteredFile filteredFile = new FilteredFile(new ByteArrayInputStream(controlFile.getBytes()), new MapVariableResolver(map), StandardCharsets.UTF_8); BinaryPackageControlFile d = new BinaryPackageControlFile(filteredFile.toString()); diff --git a/src/test/java/org/vafer/jdeb/utils/UtilsTestCase.java b/src/test/java/org/vafer/jdeb/utils/UtilsTestCase.java index 324489c08..19c95bf5c 100644 --- a/src/test/java/org/vafer/jdeb/utils/UtilsTestCase.java +++ b/src/test/java/org/vafer/jdeb/utils/UtilsTestCase.java @@ -51,7 +51,7 @@ public void testStripPath() { } private String convert(String s) throws Exception { - byte[] data = Utils.toUnixLineEndings(new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8))); + byte[] data = Utils.toUnixLineEndings(new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); return new String(data, StandardCharsets.UTF_8); } diff --git a/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/control b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/control new file mode 100644 index 000000000..9b9069257 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/control @@ -0,0 +1,13 @@ +Package: test +Version: 1.0.1 +Section: misc +Priority: optional +Architecture: i386 +Depends: some-package +Maintainer: {[{maintainer}]} +Distribution: development +Description: revision @REVISION@, test package + This is a sample package control file. + . + Use for testing purposes only. +XB-UserDefinedField: This is a user defined field. diff --git a/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/postinst b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/postinst new file mode 100644 index 000000000..9bb2140c8 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/postinst @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Post installation script for {[{name}]} {[{version}]} +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/prerm b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/prerm new file mode 100644 index 000000000..844f0dd12 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlcustomtoken/prerm @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Pre removal script for {[{name}]} {[{version}]} +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/control b/src/test/resources/org/vafer/jdeb/deb/controlencoding/control new file mode 100644 index 000000000..1666796a8 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/control @@ -0,0 +1,13 @@ +Package: test +Version: 1.0.1 +Section: misc +Priority: optional +Architecture: i386 +Depends: some-package +Maintainer: ジョン Doe +Distribution: development +Description: revision @REVISION@, test package + This is a sample package control file. + . + Use for testing purposes only. +XB-UserDefinedField: This is a user defined field. diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/postinst b/src/test/resources/org/vafer/jdeb/deb/controlencoding/postinst new file mode 100644 index 000000000..27805a131 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/postinst @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Post installation script for [[name]] [[version]] created by ジョン +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/prerm b/src/test/resources/org/vafer/jdeb/deb/controlencoding/prerm new file mode 100644 index 000000000..09d9a4313 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/prerm @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Pre removal script for [[name]] [[version]] created by ジョン +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_invalid_eol.sh b/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_invalid_eol.sh new file mode 100644 index 000000000..33f43cb42 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_invalid_eol.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Custom script created by ジョン +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_valid_eol.sh b/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_valid_eol.sh new file mode 100644 index 000000000..33f43cb42 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/shell_valid_eol.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# +# Custom script created by ジョン +# diff --git a/src/test/resources/org/vafer/jdeb/deb/controlencoding/text.txt b/src/test/resources/org/vafer/jdeb/deb/controlencoding/text.txt new file mode 100644 index 000000000..a50774a27 --- /dev/null +++ b/src/test/resources/org/vafer/jdeb/deb/controlencoding/text.txt @@ -0,0 +1 @@ +Text file created by ジョン