initial commit.

.gitignore

@@ -0,0 +1,11 @@
+*~
+*#
+*.o
+*.so
+*.dylib
+*.dSYM
+*.dll
+*.dummy
+*.exe
+build
+Makefile

LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2013, Kang Seonghoon.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

Makefile.in

@@ -0,0 +1,28 @@
+VPATH=%VPATH%
+
+CC ?= gcc
+CXX ?= g++
+CXXFLAGS ?=
+AR ?= ar
+RUSTC ?= rustc
+RUSTFLAGS ?= -O
+
+RUST_SRC=$(shell find $(VPATH)/src/. -type f -name '*.rs')
+
+.PHONY: all
+all:	librustencoding.dummy
+
+librustencoding.dummy: src/encoding.rs $(RUST_SRC)
+	$(RUSTC) $(RUSTFLAGS) $< -o $@
+	touch $@
+
+rustencoding-test: src/encoding.rs $(RUST_SRC)
+	$(RUSTC) $(RUSTFLAGS) $< -o $@ --test
+
+check: rustencoding-test
+	./rustencoding-test
+
+.PHONY: clean
+clean:
+	rm -f *.o *.a *.so *.dylib *.dll *.dummy *-test
+

README.md

@@ -0,0 +1,66 @@
+rust-encoding
+=============
+
+Character encoding support for Rust.
+It is based on [WHATWG Encoding Standard](http://encoding.spec.whatwg.org/),
+and also provides an advanced interface for error detection and recovery.
+
+Usage
+-----
+
+To encode a string:
+
+~~~~ {.rust}
+use encoding::*;
+all::ISO_8859_2.encode("caf\xe9", Strict); // => Ok(~[99,97,102,233])
+~~~~
+
+To encode a string with unrepresentable characters:
+
+~~~~ {.rust}
+use encoding::*;
+all::ISO_8859_2.encode("Acme\xa9", Strict); // => Err(...)
+all::ISO_8859_2.encode("Acme\xa9", Replace); // => Ok(~[65,99,109,101,63])
+all::ISO_8859_2.encode("Acme\xa9", Ignore); // => Ok(~[65,99,109,101])
+~~~~
+
+To decode a byte sequence:
+
+~~~~ {.rust}
+use encoding::*;
+all::ISO_8859_2.decode([99,97,102,233], Strict); // => Ok(~"caf\xe9")
+~~~~
+
+To decode a byte sequence with invalid sequences:
+
+~~~~ {.rust}
+use encoding::*;
+all::ISO_8859_6.decode([65,99,109,101,169], Strict); // => Err(...)
+all::ISO_8859_6.decode([65,99,109,101,169], Replace); // => Ok(~"Acme\ufffd")
+all::ISO_8859_6.decode([65,99,109,101,169], Ignore); // => Ok(~"Acme")
+~~~~
+
+To get an encoding from a string label:
+
+~~~~ {.rust}
+use encoding::*;
+let latin2 = label::get_encoding("Latin2").unwrap();
+latin2.name(); // => ~"iso-8859-2"
+latin2.encode("caf\xe9", Strict); // => Ok(~[99,97,102,233])
+~~~~
+
+Supported Encodings
+-------------------
+
+Rust-encoding is a work in progress and this list will certainly be updated.
+
+* 7-bit strict ASCII (`ascii`)
+* All single byte encoding in WHATWG Encoding Standard:
+    * IBM code page 866
+    * ISO-8859-{2,3,4,5,6,7,8,10,13,14,15,16}
+    * KOI8-R, KOI8-U
+    * MacRoman (`macintosh`), Macintosh Cyrillic encoding (`x-mac-cyrillic`)
+    * Windows code page 874, 1250, 1251, 1252 (instead of ISO-8859-1), 1253, 1254 (instead of ISO-8859-9), 1255, 1256, 1257, 1258
+
+Note that `label::get_encoding` does not cover every available encoding as it was designed for HTML's loose processing.
+

configure

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SRCDIR="$(cd $(dirname $0) && pwd)"
+sed "s#%VPATH%#${SRCDIR}#" ${SRCDIR}/Makefile.in > Makefile
+

src/all.rs

@@ -0,0 +1,51 @@
+// This is a part of rust-encoding.
+// Copyright (c) 2013, Kang Seonghoon.
+// See README.md and LICENSE.txt for details.
+
+//! A list of all supported encodings. Useful for encodings fixed in the compile time.
+
+use super::{index, codec};
+
+pub static ASCII: &'static codec::ascii::ASCIIEncoding =
+    &codec::ascii::ASCIIEncoding;
+
+macro_rules! singlebyte(
+    (var=$var:ident, mod=$module:ident, name=$name:expr) => (
+        pub static $var: &'static codec::singlebyte::SingleByteEncoding =
+            &codec::singlebyte::SingleByteEncoding {
+                name: $name,
+                index_forward: index::$module::forward,
+                index_backward: index::$module::backward,
+            };
+    )
+)
+
+singlebyte!(var=IBM866, mod=ibm866, name="ibm866")
+singlebyte!(var=ISO_8859_2, mod=iso_8859_2, name="iso-8859-2")
+singlebyte!(var=ISO_8859_3, mod=iso_8859_3, name="iso-8859-3")
+singlebyte!(var=ISO_8859_4, mod=iso_8859_4, name="iso-8859-4")
+singlebyte!(var=ISO_8859_5, mod=iso_8859_5, name="iso-8859-5")
+singlebyte!(var=ISO_8859_6, mod=iso_8859_6, name="iso-8859-6")
+singlebyte!(var=ISO_8859_7, mod=iso_8859_7, name="iso-8859-7")
+singlebyte!(var=ISO_8859_8, mod=iso_8859_8, name="iso-8859-8")
+singlebyte!(var=ISO_8859_8_I, mod=iso_8859_8, name="iso-8859-8-i")
+singlebyte!(var=ISO_8859_10, mod=iso_8859_10, name="iso-8859-10")
+singlebyte!(var=ISO_8859_13, mod=iso_8859_13, name="iso-8859-13")
+singlebyte!(var=ISO_8859_14, mod=iso_8859_14, name="iso-8859-14")
+singlebyte!(var=ISO_8859_15, mod=iso_8859_15, name="iso-8859-15")
+singlebyte!(var=ISO_8859_16, mod=iso_8859_16, name="iso-8859-16")
+singlebyte!(var=KOI8_R, mod=koi8_r, name="koi8-r")
+singlebyte!(var=KOI8_U, mod=koi8_u, name="koi8-u")
+singlebyte!(var=MACINTOSH, mod=macintosh, name="macintosh")
+singlebyte!(var=WINDOWS_874, mod=windows_874, name="windows-874")
+singlebyte!(var=WINDOWS_1250, mod=windows_1250, name="windows-1250")
+singlebyte!(var=WINDOWS_1251, mod=windows_1251, name="windows-1251")
+singlebyte!(var=WINDOWS_1252, mod=windows_1252, name="windows-1252")
+singlebyte!(var=WINDOWS_1253, mod=windows_1253, name="windows-1253")
+singlebyte!(var=WINDOWS_1254, mod=windows_1254, name="windows-1254")
+singlebyte!(var=WINDOWS_1255, mod=windows_1255, name="windows-1255")
+singlebyte!(var=WINDOWS_1256, mod=windows_1256, name="windows-1256")
+singlebyte!(var=WINDOWS_1257, mod=windows_1257, name="windows-1257")
+singlebyte!(var=WINDOWS_1258, mod=windows_1258, name="windows-1258")
+singlebyte!(var=X_MAC_CYRILLIC, mod=x_mac_cyrillic, name="x-mac-cyrillic")
+

src/codec/ascii.rs

@@ -0,0 +1,118 @@
+// This is a part of rust-encoding.
+// Copyright (c) 2013, Kang Seonghoon.
+// See README.md and LICENSE.txt for details.
+
+//! 7-bit ASCII encoding.
+
+use std::str;
+use util::StrCharIndex;
+use types::*;
+
+#[deriving(Clone)]
+pub struct ASCIIEncoding;
+
+impl Encoding for ASCIIEncoding {
+    pub fn name(&self) -> ~str { ~"ascii" }
+    pub fn encoder(&self) -> ~Encoder { ~ASCIIEncoder as ~Encoder }
+    pub fn decoder(&self) -> ~Decoder { ~ASCIIDecoder as ~Decoder }
+    pub fn preferred_replacement_seq(&self) -> ~[u8] { ~[0x3f] /* "?" */ }
+}
+
+#[deriving(Clone)]
+pub struct ASCIIEncoder;
+
+impl Encoder for ASCIIEncoder {
+    pub fn encoding(&self) -> ~Encoding { ~ASCIIEncoding as ~Encoding }
+
+    pub fn feed<'r>(&mut self, input: &'r str) -> (~[u8],Option<EncoderError<'r>>) {
+        let mut ret = ~[];
+        let mut err = None;
+        for input.index_iter().advance |((_,j), ch)| {
+            if ch <= '\u007f' {
+                ret.push(ch as u8);
+            } else {
+                err = Some(CodecError {
+                    remaining: input.slice_from(j),
+                    problem: str::from_char(ch),
+                    cause: ~"unrepresentable character",
+                });
+                break;
+            }
+        }
+        (ret, err)
+    }
+
+    pub fn flush(~self) -> (~[u8],Option<EncoderError<'static>>) {
+        (~[], None)
+    }
+}
+
+#[deriving(Clone)]
+pub struct ASCIIDecoder;
+
+impl Decoder for ASCIIDecoder {
+    pub fn encoding(&self) -> ~Encoding { ~ASCIIEncoding as ~Encoding }
+
+    pub fn feed<'r>(&mut self, input: &'r [u8]) -> (~str,Option<DecoderError<'r>>) {
+        let mut ret = ~"";
+        let mut i = 0;
+        let len = input.len();
+        while i < len {
+            if input[i] <= 0x7f {
+                ret.push_char(input[i] as char);
+            } else {
+                return (ret, Some(CodecError {
+                    remaining: input.slice(i+1, input.len()),
+                    problem: ~[input[i]],
+                    cause: ~"invalid sequence",
+                }));
+            }
+            i += 1;
+        }
+        (ret, None)
+    }
+
+    pub fn flush(~self) -> (~str,Option<DecoderError<'static>>) {
+        (~"", None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ASCIIEncoding;
+    use types::*;
+
+    fn strip_cause<T,Remaining,Problem>(result: (T,Option<CodecError<Remaining,Problem>>))
+                                    -> (T,Option<(Remaining,Problem)>) {
+        match result {
+            (processed, None) => (processed, None),
+            (processed, Some(CodecError { remaining, problem, cause: _cause })) =>
+                (processed, Some((remaining, problem)))
+        }
+    }
+
+    macro_rules! assert_result(
+        ($lhs:expr, $rhs:expr) => (assert_eq!(strip_cause($lhs), $rhs))
+    )
+
+    #[test]
+    fn test_encoder() {
+        let mut e = ASCIIEncoding.encoder();
+        assert_result!(e.feed("A"), (~[0x41], None));
+        assert_result!(e.feed("BC"), (~[0x42, 0x43], None));
+        assert_result!(e.feed(""), (~[], None));
+        assert_result!(e.feed("\xa0"), (~[], Some(("", ~"\xa0"))));
+        assert_result!(e.flush(), (~[], None));
+    }
+
+    #[test]
+    fn test_decoder() {
+        let mut d = ASCIIEncoding.decoder();
+        assert_result!(d.feed(&[0x41]), (~"A", None));
+        assert_result!(d.feed(&[0x42, 0x43]), (~"BC", None));
+        assert_result!(d.feed(&[]), (~"", None));
+        assert_result!(d.feed(&[0xa0]), (~"", Some((&[], ~[0xa0]))));
+        assert_result!(d.flush(), (~"", None));
+    }
+}
+