Merge branch 'master' into v4_experiments

# Conflicts:
#	pom.xml
#	src/test/java/org/sejda/sambox/pdmodel/interactive/form/AppearanceGeneratorHelperTest.java

Author: torakiki

src/main/java/org/sejda/sambox/cos/PDFDocEncoding.java

@@ -53,10 +53,6 @@ final class PDFDocEncoding
             {
                 continue;
             }
-            if (i == 0xAD)
-            {
-                continue;
-            }
 
             set(i, (char) i);
         }

src/main/java/org/sejda/sambox/pdmodel/interactive/form/AppearanceGeneratorHelper.java

@@ -664,7 +664,7 @@ else if (field instanceof PDListBox)
         }
         else
         {
-            PlainText textContent = new PlainText(value);
+            PlainText textContent = new PlainText(normalizeWhitespace(value));
             AppearanceStyle appearanceStyle = new AppearanceStyle();
             appearanceStyle.setFont(font);
             appearanceStyle.setFontSize(fontSize);
@@ -993,7 +993,8 @@ float calculateFontSize(PDFont font, PDRectangle contentRect) throws IOException
         }
 
         // fit width
-        float width = font.getStringWidth(value) * font.getFontMatrix().getScaleX();
+        float width = font.getStringWidth(normalizeWhitespace(stripNonPrintableChars(value)))
+                * font.getFontMatrix().getScaleX();
         float widthBasedFontSize = contentRect.getWidth() / width * xScalingFactor;
 
         // fit height
@@ -1004,6 +1005,19 @@ float calculateFontSize(PDFont font, PDRectangle contentRect) throws IOException
         return Math.min(heightBasedFontSize, widthBasedFontSize);
     }
 
+    private String stripNonPrintableChars(String s)
+    {
+        // TODO: add more non printable chars
+        return s.replaceAll("\\u000D", ""); // carriage return \r
+    }
+
+    private String normalizeWhitespace(String s)
+    {
+        if (s == null)
+            return null;
+        return s.replaceAll("\\p{Zs}", " ");
+    }
+
     /*
      * Resolve the cap height.
      *

src/test/java/org/sejda/sambox/cos/PDFDocEncodingTest.java

@@ -17,6 +17,7 @@
 
 package org.sejda.sambox.cos;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
@@ -132,4 +133,64 @@ public void testPDFBox3864() throws IOException
             assertEquals(cs1.getString(), cs2.getString());
         }
     }
+
+    /**
+     * 0xAD (SOFT HYPHEN) must decode to U+00AD, not U+0000. ISO 32000-1 leaves it undefined, but
+     * other producers treat it as its ISO-8859-1 value; decoding it to a NUL control character
+     * corrupts text strings such as form field names.
+     */
+    @Test
+    public void softHyphenDecodesToU00AD()
+    {
+        assertEquals("\u00AD", PDFDocEncoding.toString(new byte[] { (byte) 0xAD }));
+    }
+
+    @Test
+    public void softHyphenEncodesTo0xAD()
+    {
+        assertArrayEquals(new byte[] { (byte) 0xAD }, PDFDocEncoding.getBytes("\u00AD"));
+    }
+
+    @Test
+    public void softHyphenRoundTrips()
+    {
+        byte[] bytes = { (byte) 0xAD };
+        assertArrayEquals(bytes, PDFDocEncoding.getBytes(PDFDocEncoding.toString(bytes)));
+    }
+
+    /**
+     * Reproduces the real-world failure: a German field name with two soft hyphens and an a-umlaut,
+     * stored as PDFDocEncoding, survives a literal round-trip via COSString and is stored
+     * single-byte (not UTF-16BE).
+     */
+    @Test
+    public void fieldNameWithSoftHyphensRoundTrips()
+    {
+        String fieldName = "Test\u00ADbesch\u00E4test adas Test\u00ADdsadas";
+        COSString cosString = COSString.parseLiteral(fieldName);
+        assertEquals(fieldName, cosString.getString());
+        // representable in PDFDocEncoding, so single-byte; UTF-16BE would be 2*len + 2 (BOM)
+        assertEquals(fieldName.length(), cosString.getBytes().length);
+    }
+
+    /**
+     * Bytes genuinely undefined in PDFDocEncoding (0x7F, 0x9F) must decode to the replacement
+     * character, never to a silent U+0000.
+     */
+    @Test
+    public void undefinedBytesDecodeToReplacementCharacter()
+    {
+        assertEquals("\uFFFD", PDFDocEncoding.toString(new byte[] { (byte) 0x7F }));
+        assertEquals("\uFFFD", PDFDocEncoding.toString(new byte[] { (byte) 0x9F }));
+    }
+
+    /**
+     * Code 0x00 is a legitimate mapping to U+0000 and must not be clobbered by the
+     * replacement-character pre-fill.
+     */
+    @Test
+    public void nulByteStillDecodesToU0000()
+    {
+        assertEquals("\u0000", PDFDocEncoding.toString(new byte[] { 0x00 }));
+    }
 }

src/test/java/org/sejda/sambox/pdmodel/interactive/form/AppearanceGeneratorHelperTest.java

@@ -3,6 +3,7 @@
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -114,13 +115,73 @@ public void testInvalidAppearanceString() throws IOException
         helper.setAppearanceValue("Some text");
     }
 
+    @Test
+    public void testCalculateFontSizeWithNonPrintableChars() throws IOException
+    {
+        PDTextField field = multilineTextFieldFaux();
+
+        AppearanceGeneratorHelper helper = new AppearanceGeneratorHelper(field);
+        helper.setAppearanceValue("This isnt multiple lines\r");
+
+        assertNotNull(field.getWidgets().get(0).getNormalAppearanceStream());
+    }
+
+    @Test
+    public void testCalculateFontSizeWithExoticWhiteSpace() throws IOException
+    {
+        PDTextField field = multilineTextFieldFaux();
+
+        AppearanceGeneratorHelper helper = new AppearanceGeneratorHelper(field);
+        helper.setAppearanceValue("This is\u2002whitespace");
+
+        assertNotNull(field.getWidgets().get(0).getNormalAppearanceStream());
+    }
+
+    @Test
+    public void testComboBoxAppearanceWithExoticWhiteSpace() throws IOException
+    {
+        PDComboBox field = comboBox();
+
+        // U+2002 (en space) has no glyph in Helvetica / WinAnsiEncoding. Before it is
+        // normalized to a regular space, generating the combo box appearance threw
+        // IllegalArgumentException from PDFont.encode (reached via getStringWidth in
+        // calculateFontSize, and via showText in the rendering path).
+        //
+        // setValue drives the exact path from the original crash report:
+        // PDChoice.setValue -> applyChange -> constructAppearances -> setAppearanceValue
+        field.setValue("Option\u2002One");
+
+        // appearance generation must complete and produce a stream, not silently skip
+        assertNotNull(field.getWidgets().get(0).getNormalAppearanceStream());
+    }
+
+    private PDComboBox comboBox()
+    {
+        PDDocument doc = new PDDocument();
+        doc.addPage(new PDPage());
+        PDAcroForm form = new PDAcroForm(doc);
+        PDComboBox cb = new PDComboBox(form);
+        cb.setDefaultAppearance("/Helvetica 10.00 Tf 0 g");
+        cb.getWidgets().get(0).setRectangle(new PDRectangle(2, 2, 100, 20));
+        return cb;
+    }
+
     private PDTextField multilineTextField()
     {
         PDTextField tf = textField();
         tf.setMultiline(true);
         return tf;
     }
 
+    private PDTextField multilineTextFieldFaux()
+    {
+        PDTextField field = multilineTextField();
+        // faux multiline, content rectangle not tall enough
+        field.getWidgets().get(0).setRectangle(new PDRectangle(2, 2, 517, 10));
+        field.setDefaultAppearance("Helvetica 0 Tf 0 g");
+        return field;
+    }
+
     private PDTextField textField()
     {
         PDDocument doc = new PDDocument();