{"action":"created","comment":{"url":"https://api.github.com/repos/JuliaLang/julia/pulls/comments/31384056","id":31384056,"diff_hunk":"@@ -0,0 +1,969 @@\n+# This file is a part of Julia. License is MIT: http://julialang.org/license\n+\n+#=\n+@doc \"\"\"\n+@brief      Error messages for Unicode / UTF support\n+\"\"\" ->\n+=#\n+\n+const UTF_ERR_SHORT = 1\n+const UTF_ERR_CONT  = 2\n+const UTF_ERR_LONG  = 3\n+const UTF_ERR_NOT_LEAD = 4\n+const UTF_ERR_NOT_TRAIL = 5\n+const UTF_ERR_NOT_SURROGATE = 6\n+const UTF_ERR_MISSING_SURROGATE = 7\n+const UTF_ERR_INVALID = 8\n+const UTF_ERR_SURROGATE = 9\n+const UTF_ERR_NULL_16_TERMINATE = 10\n+const UTF_ERR_NULL_32_TERMINATE = 11\n+const UTF_ERR_MAX = 11\n+\n+const errMsgs = [\n+    \"invalid UTF-8 sequence starting at index <<1>> (0x<<2>>) missing one or more continuation bytes)\",\n+    \"invalid UTF-8 sequence starting at index <<1>> (0x<<2>> is not a continuation byte)\",\n+    \"invalid UTF-8 sequence, overlong encoding starting at index <<1>> (0x<<2>>)\",\n+    \"not a leading Unicode surrogate character at index <<1>> (0x<<2>>)\",\n+    \"not a trailing Unicode surrogate character at index <<1>> (0x<<2>>)\",\n+    \"not a valid Unicode surrogate character at index <<1>> (0x<<2>>\",\n+    \"missing trailing Unicode surrogate character after index <<1>> (0x<<2>>)\",\n+    \"invalid Unicode character starting at index <<1>> (0x<<2>> > 0x10ffff)\",\n+    \"surrogate encoding not allowed in UTF-8 or UTF-32, at index <<1>> (0x<<2>>)\",\n+    \"UTF16String data must be NULL-terminated\",\n+    \"UTF32String data must be NULL-terminated\"\n+]\n+#=\n+@doc \"\"\"\n+@brief      Throws ArgumentError with information about the specific error, location, and character\n+\n+@param[in]  errcode Error code for Unicode error (one of UTF_ERR_*)\n+@param[in]  charpos Index of invalid byte or character\n+@param[in]  invchar Invalid byte or character\n+\n+@throws never returns, always throws ArgumentError\n+\"\"\" ->\n+=#\n+function utf_errfunc(errcode::Integer, charpos, invchar)\n+    if errcode < 1 || errcode > UTF_ERR_MAX\n+        throw(ArgumentError(\"Invalid error code for Unicode error: $errcode, Pos = $charpos, Char = $invchar\"))\n+    end\n+    throw(ArgumentError(replace(replace(errMsgs[errcode],\"<<1>>\",string(charpos)),\"<<2>>\",hex(invchar))))\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Base UTF16String type, has 16-bit NULL termination word after data, native byte order\n+\"\"\" ->\n+=#\n+immutable UTF16String <: AbstractString\n+    data::Vector{UInt16} # includes 16-bit NULL termination after string chars\n+    function UTF16String(data::Vector{UInt16})\n+        if length(data) < 1 || data[end] != 0\n+            utf_errfunc(UTF_ERR_NULL_16_TERMINATE, 0, 0)\n+        end\n+        new(data)\n+    end\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Base UTF32String type, has 32-bit NULL termination word after data, native byte order\n+\"\"\" ->\n+=#\n+immutable UTF32String <: DirectIndexString\n+    data::Vector{Char} # includes 32-bit NULL termination after string chars\n+\n+    function UTF32String(data::Vector{Char})\n+        if length(data) < 1 || data[end] != Char(0)\n+            utf_errfunc(UTF_ERR_NULL_32_TERMINATE, 0, 0)\n+        end\n+        new(data)\n+    end\n+end\n+UTF32String(data::Vector{UInt32}) = UTF32String(reinterpret(Char, data))\n+\n+const empty_utf16 = UTF16String(UInt16[0])\n+const empty_utf32 = UTF32String(UInt32[0])\n+\n+is_surrogate_lead(c::Unsigned) = ((c & ~0x003ff) == 0xd800)\n+is_surrogate_trail(c::Unsigned) = ((c & ~0x003ff) == 0xdc00)\n+is_surrogate_char(c::Unsigned) = ((c & ~0x007ff) == 0xd800)\n+is_valid_continuation(c) = ((c & 0xc0) == 0x80)\n+\n+function length(s::UTF16String)\n+    d = s.data\n+    len = length(d) - 1\n+    len == 0 && return 0\n+    cnum = 0\n+    for i = 1:len\n+        @inbounds cnum += !is_surrogate_trail(d[i])\n+    end\n+    cnum\n+end\n+\n+function endof(s::UTF16String)\n+    d = s.data\n+    i = length(d) - 1\n+    i == 0 && return i\n+    return is_surrogate_char(d[i]) ? i-1 : i\n+end\n+\n+get_supplementary(lead::Unsigned, trail::Unsigned) = (UInt32(lead-0xd7f7)<<10 + trail)\n+\n+function next(s::UTF16String, i::Int)\n+    ch = s.data[i]\n+    !is_surrogate_char(ch) && return (Char(ch), i+1)\n+    # check length, account for terminating \\0\n+    i >= (length(s.data)-1) && utf_errfunc(UTF_ERR_MISSING_SURROGATE, i, UInt32(ch))\n+    !is_surrogate_lead(ch) && utf_errfunc(UTF_ERR_NOT_LEAD, i, ch)\n+    ct = s.data[i+1]\n+    !is_surrogate_trail(ct) && utf_errfunc(UTF_ERR_NOT_TRAIL, i, ch)\n+    Char(get_supplementary(ch, ct)), i+2\n+end\n+\n+function reverseind(s::UTF16String, i::Integer)\n+    j = length(s.data) - i\n+    return is_surrogate_trail(s.data[j]) ? j-1 : j\n+end\n+\n+lastidx(s::UTF16String) = length(s.data) - 1 # s.data includes NULL terminator\n+\n+function reverse(s::UTF16String)\n+    d = s.data\n+    out = similar(d)\n+    out[end] = 0 # NULL termination\n+    n = length(d)\n+    @inbounds for i = 1:n-1\n+        ch = d[n-i]\n+        if is_surrogate_lead(ch)\n+            out[i],out[i-1] = out[i-1],ch\n+        else\n+            out[i] = ch\n+        end\n+    end\n+    UTF16String(out)\n+end\n+\n+next(s::UTF32String, i::Int) = (s.data[i], i+1)\n+endof(s::UTF32String) = length(s.data) - 1\n+length(s::UTF32String) = length(s.data) - 1\n+\n+const UTF_NO_LONG_NULL = 1      # don't accept 0xc0 0x80 for '\\0'\n+const UTF_NO_SURROGATES = 2     # don't accept surrogate pairs in UTF-8/UTF-32\n+const UTF_ACCEPT_LONG = 4       # accept long encodings (other than long null in UTF-8)\n+\n+const UTF_LONG = 1              # Long encodings are present\n+const UTF_LATIN1 = 2            # characters in range 0x80-0xFF present\n+const UTF_UNICODE2 = 4          # characters in range 0x100-0x7ff present\n+const UTF_UNICODE3 = 8          # characters in range 0x800-0xd7ff, 0xe000-0xffff\n+const UTF_UNICODE4 = 16         # non-BMP characters present\n+const UTF_SURROGATE = 32        # surrogate pairs present\n+\n+# Get a UTF-8 continuation byte, give error if invalid, and update position and character value\n+@inline function get_continuation(ch::UInt32, str, pos)\n+    byt::UInt8 = str[pos += 1]\n+    !is_valid_continuation(byt) && utf_errfunc(UTF_ERR_CONT, pos, byt)\n+    (ch << 6) | (byt & 0x3f), pos\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Validates and calculates number of characters in a string\n+\n+@param[in]  str     Vector of UInt8\n+@param[in]  options flags to determine error handling (default 0)\n+\n+@return     (total characters, flags, 4-byte, 3-byte, 2-byte)\n+@throws     ArgumentError\n+\"\"\" ->\n+=#\n+function check_string_utf8(dat::Vector{UInt8}, options::Integer=0)\n+    local byt::UInt8, ch::UInt32, surr::UInt32\n+    flags::UInt = 0\n+    totalchar = num2byte = num3byte = num4byte = 0\n+    pos = 0\n+    len = sizeof(dat)\n+    @inbounds while pos < len\n+        ch = dat[pos += 1]\n+        totalchar += 1\n+        if ch > 0x7f\n+            # Check UTF-8 encoding\n+            if ch < 0xe0\n+                # 2-byte UTF-8 sequence (i.e. characters 0x80-0x7ff)\n+                (pos == len) && utf_errfunc(UTF_ERR_SHORT, pos, ch)\n+                ch, pos = get_continuation(ch & 0x3f, dat, pos)\n+                if ch > 0x7f\n+                    num2byte += 1\n+                    flags |= (ch > 0xff) ? UTF_UNICODE2 : UTF_LATIN1\n+                elseif (options & UTF_ACCEPT_LONG) != 0\n+                    flags |= UTF_LONG\n+                elseif (ch == 0) && ((options & UTF_NO_LONG_NULL) == 0)\n+                    flags |= UTF_LONG\n+                else\n+                    utf_errfunc(UTF_ERR_LONG, pos, ch)\n+                end\n+             elseif ch < 0xf0\n+                # 3-byte UTF-8 sequence (i.e. characters 0x800-0xffff)\n+                (pos + 2 > len) && utf_errfunc(UTF_ERR_SHORT, pos, ch)\n+                ch, pos = get_continuation(ch & 0x0f, dat, pos)\n+                ch, pos = get_continuation(ch, dat, pos)\n+                # check for surrogate pairs, make sure correct\n+                if is_surrogate_char(ch)\n+                    !is_surrogate_lead(ch) && utf_errfunc(UTF_ERR_NOT_LEAD, pos-2, ch)\n+                    # next character *must* be a trailing surrogate character\n+                    (pos + 3 > len) && utf_errfunc(UTF_ERR_MISSING_SURROGATE, pos-2, ch)\n+                    byt = dat[pos += 1]\n+                    (byt != 0xed) && utf_errfunc(UTF_ERR_NOT_TRAIL, pos, byt)\n+                    surr, pos = get_continuation(0x0000d, dat, pos)\n+                    surr, pos = get_continuation(surr, dat, pos)\n+                    !is_surrogate_trail(surr) && utf_errfunc(UTF_ERR_NOT_TRAIL, pos-2, surr)\n+                    (options & UTF_NO_SURROGATES) != 0 && utf_errfunc(UTF_ERR_SURROGATE, pos-2, surr)\n+                    flags |= UTF_SURROGATE\n+                    num4byte += 1\n+                elseif ch > 0x07ff\n+                    num3byte += 1\n+                elseif (options & UTF_ACCEPT_LONG) != 0\n+                    flags |= UTF_LONG\n+                    num2byte += 1\n+                else\n+                    utf_errfunc(UTF_ERR_LONG, pos-2, ch)\n+                end\n+            elseif ch < 0xf5\n+                # 4-byte UTF-8 sequence (i.e. characters > 0xffff)\n+                (pos + 3 > len) && utf_errfunc(UTF_ERR_SHORT, pos, ch)\n+                ch, pos = get_continuation(ch & 0x07, dat, pos)\n+                ch, pos = get_continuation(ch, dat, pos)\n+                ch, pos = get_continuation(ch, dat, pos)\n+                if ch > 0x10ffff\n+                    utf_errfunc(UTF_ERR_INVALID, pos-3, ch)\n+                elseif ch > 0xffff\n+                    num4byte += 1\n+                elseif is_surrogate_char(ch)\n+                    utf_errfunc(UTF_ERR_SURROGATE, pos-3, ch)\n+                elseif (options & UTF_ACCEPT_LONG) != 0\n+                    # This is an overly long encode character\n+                    flags |= UTF_LONG\n+                    if ch > 0x7ff\n+                        num3byte += 1\n+                    elseif ch > 0x7f\n+                        num2byte += 1\n+                    end\n+                else\n+                    utf_errfunc(UTF_ERR_LONG, pos-2, ch)\n+                end\n+            else\n+                utf_errfunc(UTF_ERR_INVALID, pos, ch)\n+            end\n+        end\n+    end\n+    num3byte != 0 && (flags |= UTF_UNICODE3)\n+    num4byte != 0 && (flags |= UTF_UNICODE4)\n+    return totalchar, flags, num4byte, num3byte, num2byte\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Validates and calculates number of characters in a UTF-16 string\n+\n+@param[in]  dat     Vector{UInt16}\n+@param[in]  options flags to determine error handling (default 0)\n+\n+@return     (total characters, flags, 4-byte, 3-byte, 2-byte)\n+@throws     ArgumentError\n+\"\"\" ->\n+=#\n+function check_string_utf16(dat::Vector{UInt16}, len::Int)\n+    local ch::UInt32\n+    flags::UInt = 0\n+    totalchar = num2byte = num3byte = num4byte = 0\n+    pos = 0\n+    @inbounds while pos < len\n+        ch = dat[pos += 1]\n+        totalchar += 1\n+        if ch > 0x7f\n+            if ch < 0x100\n+                num2byte += 1\n+                flags |= UTF_LATIN1\n+            elseif ch < 0x800\n+                num2byte += 1\n+                flags |= UTF_UNICODE2\n+            elseif !is_surrogate_char(ch)\n+                num3byte += 1\n+            elseif is_surrogate_lead(ch)\n+                pos == len && utf_errfunc(UTF_ERR_MISSING_SURROGATE, pos, ch)\n+                # next character *must* be a trailing surrogate character\n+                ch = dat[pos += 1]\n+                !is_surrogate_trail(ch) && utf_errfunc(UTF_ERR_NOT_TRAIL, pos, ch)\n+                num4byte += 1\n+            else\n+                utf_errfunc(UTF_ERR_NOT_LEAD, pos, ch)\n+            end\n+        end\n+    end\n+    num3byte != 0 && (flags |= UTF_UNICODE3)\n+    num4byte != 0 && (flags |= UTF_UNICODE4)\n+    return totalchar, flags, num4byte, num3byte, num2byte\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Validates and calculates number of characters in a UTF-32 string\n+\n+@param[in]  dat     Vector{UInt32}\n+@param[in]  options flags to determine error handling (default 0)\n+\n+@return     (total characters, flags, 4-byte, 3-byte, 2-byte)\n+@throws     ArgumentError\n+\"\"\" ->\n+=#\n+function check_string_utf32(dat::Vector{UInt32}, len::Int, options::Integer=0)\n+    local ch::UInt32\n+    flags::UInt = 0\n+    totalchar = num2byte = num3byte = num4byte = 0\n+    pos = 0\n+    @inbounds while pos < len\n+        ch = dat[pos += 1]\n+        totalchar += 1\n+        if ch > 0x7f\n+            if ch < 0x100\n+                num2byte += 1\n+                flags |= UTF_LATIN1\n+            elseif ch < 0x800\n+                num2byte += 1\n+                flags |= UTF_UNICODE2\n+            elseif ch > 0xffff\n+                (ch > 0x10ffff) && utf_errfunc(UTF_ERR_INVALID, pos, ch)\n+                num4byte += 1\n+            elseif !is_surrogate_char(ch)\n+                num3byte += 1\n+            elseif is_surrogate_lead(ch)\n+                pos == len && utf_errfunc(UTF_ERR_MISSING_SURROGATE, pos, ch)\n+                # next character *must* be a trailing surrogate character\n+                ch = dat[pos += 1]\n+                !is_surrogate_trail(ch) && utf_errfunc(UTF_ERR_NOT_TRAIL, pos, ch)\n+                num4byte += 1\n+                (options & UTF_NO_SURROGATES) != 0 && utf_errfunc(UTF_ERR_SURROGATE, pos, ch)\n+                flags |= UTF_SURROGATE\n+            else\n+                utf_errfunc(UTF_ERR_NOT_LEAD, pos, ch)\n+            end\n+        end\n+    end\n+    num3byte != 0 && (flags |= UTF_UNICODE3)\n+    num4byte != 0 && (flags |= UTF_UNICODE4)\n+    return totalchar, flags, num4byte, num3byte, num2byte\n+end\n+\n+function check_string_abs(str::AbstractString, options::Integer=0)\n+    local ch::UInt32\n+    flags::UInt = 0\n+    totalchar = num2byte = num3byte = num4byte = 0\n+    pos = start(str)\n+    len = endof(str)\n+    @inbounds while pos < len\n+        ch, pos = next(str, pos)\n+        totalchar += 1\n+        if ch > 0x7f\n+            if ch < 0x100\n+                num2byte += 1\n+                flags |= UTF_LATIN1\n+            elseif ch < 0x800\n+                num2byte += 1\n+                flags |= UTF_UNICODE2\n+            elseif ch > 0xffff\n+                (ch > 0x10ffff) && utf_errfunc(UTF_ERR_INVALID, pos, ch)\n+                num4byte += 1\n+            elseif !is_surrogate_char(ch)\n+                num3byte += 1\n+            elseif is_surrogate_lead(ch)\n+                pos == len && utf_errfunc(UTF_ERR_MISSING_SURROGATE, pos, ch)\n+                # next character *must* be a trailing surrogate character\n+                ch, pos = next(str, pos)\n+                !is_surrogate_trail(ch) && utf_errfunc(UTF_ERR_NOT_TRAIL, pos, ch)\n+                num4byte += 1\n+                (options & UTF_NO_SURROGATES) != 0 && utf_errfunc(UTF_ERR_SURROGATE, pos, ch)\n+                flags |= UTF_SURROGATE\n+            else\n+                utf_errfunc(UTF_ERR_NOT_LEAD, pos, ch)\n+            end\n+        end\n+    end\n+    num3byte != 0 && (flags |= UTF_UNICODE3)\n+    num4byte != 0 && (flags |= UTF_UNICODE4)\n+    return totalchar, flags, num4byte, num3byte, num2byte\n+end\n+\n+# Quickly copy and set trailing \\0\n+@inline function fast_utf_copy(T::Type{UInt16}, len, dat)\n+    @inbounds return UTF16String(setindex!(copy!(Vector{T}(len), dat), 0, len))\n+end\n+@inline function fast_utf_copy(T::Type{Char}, len, dat)\n+    @inbounds return UTF32String(setindex!(copy!(Vector{T}(len), dat), 0, len))\n+end\n+\n+# Get rest of character ch from 3-byte UTF-8 sequence in str\n+@inline function get_utf8_3(dat, pos, ch)\n+    @inbounds return ((ch & 0xf) << 12) | (UInt32(dat[pos-1] & 0x3f) << 6) | (dat[pos] & 0x3f)\n+end\n+\n+# Get rest of character ch from 4-byte UTF-8 sequence in dat, update pos and return character\n+@inline function get_utf8_4(dat, pos, ch)\n+    @inbounds return ((ch & 0x7) << 18) | (UInt32(dat[pos-2] & 0x3f) << 12) | (UInt32(dat[pos-1] & 0x3f) << 6) | (dat[pos] & 0x3f)\n+end\n+\n+# Output a character as a 4-byte UTF-8 sequence, update the position\n+@inline function output_utf8_4(buf, out, ch)\n+    @inbounds begin\n+        buf[out + 1] = 0xf0 | (ch >>> 18)\n+        buf[out + 2] = 0x80 | ((ch >>> 12) & 0x3f)\n+        buf[out + 3] = 0x80 | ((ch >>> 6) & 0x3f)\n+        buf[out + 4] = 0x80 | (ch & 0x3f)\n+    end\n+end\n+\n+#=\n+\"\"\"\n+@brief      Converts an AbstractString to a UTF16String\n+\n+@param[in]  ::Type{UTF16String}\n+@param[in]  str::AbstractString\n+\n+@return     ::UTF16String\n+@throws     ArgumentError\n+\"\"\"\n+=#\n+function convert(::Type{UTF16String}, str::AbstractString)\n+    len, flags, num4byte = check_string_abs(str)\n+    buf = Vector{UInt16}(len+num4byte+1)\n+    out = 0\n+    @inbounds for ch in str\n+        c = reinterpret(UInt32, ch)\n+        if c < 0x10000\n+            buf[out += 1] = UInt16(c)\n+        else\n+            # output surrogate pair\n+            buf[out += 1] = UInt16(0xd7c0 + (ch >>> 10))\n+            buf[out += 1] = UInt16(0xdc00 + (ch & 0x3ff))\n+        end\n+    end\n+    @inbounds buf[out + 1] = 0 # NULL termination\n+    UTF16String(buf)\n+end\n+\n+#=\n+\"\"\"\n+@brief      Converts an AbstractString to a UTF32String\n+\n+@param[in]  ::Type{UTF32String}\n+@param[in]  str::AbstractString\n+\n+@return     ::UTF32String\n+@throws     ArgumentError\n+\"\"\"\n+=#\n+function convert(::Type{UTF32String}, str::AbstractString)\n+    len, flags = check_string_abs(str)\n+    buf = Vector{Char}(len+1)\n+    out = 0\n+    @inbounds for ch in str ; buf[out += 1] = ch ; end\n+    @inbounds buf[out + 1] = 0 # NULL termination\n+    UTF32String(buf)\n+end\n+\n+#=\n+@doc \"\"\"\n+@brief      Converts a UTF-8 encoded string to UTF-16 encoding\n+\n+@param[in]  str::Vector{UInt8}\n+\n+@return     ::UTF16String\n+@throws     ArgumentError\n+\"\"\" ->\n+=#\n+function convert(::Type{UTF16String}, str::UTF8String)\n+    dat = str.data\n+    # handle zero length string quickly\n+    sizeof(dat) == 0 && return empty_utf16\n+    # Check that is correct UTF-8 encoding and get number of words needed\n+    len, flags, num4byte = check_string_utf8(dat)\n+    len += num4byte\n+    buf = Vector{UInt16}(len+1)\n+    buf[len+1] = 0\n+    # Optimize case where no characters > 0x7f\n+    flags == 0 && @inbounds return UTF16String(copy!(buf, dat))","path":"base/utf.jl","position":493,"original_position":493,"commit_id":"d808ed49b0e2d24a13cf87948b37c148adce7ce2","original_commit_id":"d808ed49b0e2d24a13cf87948b37c148adce7ce2","user":{"login":"nalimilan","id":1120448,"avatar_url":"https://avatars.githubusercontent.com/u/1120448?v=3","gravatar_id":"","url":"https://api.github.com/users/nalimilan","html_url":"https://github.com/nalimilan","followers_url":"https://api.github.com/users/nalimilan/followers","following_url":"https://api.github.com/users/nalimilan/following{/other_user}","gists_url":"https://api.github.com/users/nalimilan/gists{/gist_id}","starred_url":"https://api.github.com/users/nalimilan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nalimilan/subscriptions","organizations_url":"https://api.github.com/users/nalimilan/orgs","repos_url":"https://api.github.com/users/nalimilan/repos","events_url":"https://api.github.com/users/nalimilan/events{/privacy}","received_events_url":"https://api.github.com/users/nalimilan/received_events","type":"User","site_admin":false},"body":"(I also spotted several other cases below.)","created_at":"2015-05-30T20:12:49Z","updated_at":"2015-05-30T20:12:49Z","html_url":"https://github.com/JuliaLang/julia/pull/11004#discussion_r31384056","pull_request_url":"https://api.github.com/repos/JuliaLang/julia/pulls/11004","_links":{"self":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/comments/31384056"},"html":{"href":"https://github.com/JuliaLang/julia/pull/11004#discussion_r31384056"},"pull_request":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/11004"}}},"pull_request":{"url":"https://api.github.com/repos/JuliaLang/julia/pulls/11004","id":34102845,"html_url":"https://github.com/JuliaLang/julia/pull/11004","diff_url":"https://github.com/JuliaLang/julia/pull/11004.diff","patch_url":"https://github.com/JuliaLang/julia/pull/11004.patch","issue_url":"https://api.github.com/repos/JuliaLang/julia/issues/11004","number":11004,"state":"open","locked":false,"title":"Fix Unicode bugs with UTF-16/UTF-32 conversions (#10959)","user":{"login":"ScottPJones","id":11798751,"avatar_url":"https://avatars.githubusercontent.com/u/11798751?v=3","gravatar_id":"","url":"https://api.github.com/users/ScottPJones","html_url":"https://github.com/ScottPJones","followers_url":"https://api.github.com/users/ScottPJones/followers","following_url":"https://api.github.com/users/ScottPJones/following{/other_user}","gists_url":"https://api.github.com/users/ScottPJones/gists{/gist_id}","starred_url":"https://api.github.com/users/ScottPJones/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScottPJones/subscriptions","organizations_url":"https://api.github.com/users/ScottPJones/orgs","repos_url":"https://api.github.com/users/ScottPJones/repos","events_url":"https://api.github.com/users/ScottPJones/events{/privacy}","received_events_url":"https://api.github.com/users/ScottPJones/received_events","type":"User","site_admin":false},"body":"Rewrote a number of the conversions between ASCIIString, UTF8String,\r\nUTF16String, and UTF32String, and also rewrote length() for\r\nUTF16String().\r\n\r\nAdded validation functions that can be used in many other places, which also fix a number of Unicode handling bugs, such as detecting overlong encodings, out of range characters, singleton surrogate characters, and missing continuation characters, among other problems.\r\n\r\nAdded over 150 lines of testing code to detect the above conversion problems\r\n\r\nAdded (in a gist) code to show other conversion problems not yet fixed:\r\nhttps://gist.github.com/ScottPJones/4e6e8938f0559998f9fc\r\n\r\nAdded (in a gist) code to benchmark the performance, to ensure that adding the extra validity\r\nchecking did not adversely affect performance (in fact, performance was greatly improved).\r\nhttps://gist.github.com/ScottPJones/79ed895f05f85f333d84","created_at":"2015-04-25T08:18:36Z","updated_at":"2015-05-30T20:12:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c6184104e4b63e30c6817fd2b892fd113d3814ef","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/JuliaLang/julia/pulls/11004/commits","review_comments_url":"https://api.github.com/repos/JuliaLang/julia/pulls/11004/comments","review_comment_url":"https://api.github.com/repos/JuliaLang/julia/pulls/comments{/number}","comments_url":"https://api.github.com/repos/JuliaLang/julia/issues/11004/comments","statuses_url":"https://api.github.com/repos/JuliaLang/julia/statuses/d808ed49b0e2d24a13cf87948b37c148adce7ce2","head":{"label":"ScottPJones:spj/fast_utf","ref":"spj/fast_utf","sha":"d808ed49b0e2d24a13cf87948b37c148adce7ce2","user":{"login":"ScottPJones","id":11798751,"avatar_url":"https://avatars.githubusercontent.com/u/11798751?v=3","gravatar_id":"","url":"https://api.github.com/users/ScottPJones","html_url":"https://github.com/ScottPJones","followers_url":"https://api.github.com/users/ScottPJones/followers","following_url":"https://api.github.com/users/ScottPJones/following{/other_user}","gists_url":"https://api.github.com/users/ScottPJones/gists{/gist_id}","starred_url":"https://api.github.com/users/ScottPJones/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScottPJones/subscriptions","organizations_url":"https://api.github.com/users/ScottPJones/orgs","repos_url":"https://api.github.com/users/ScottPJones/repos","events_url":"https://api.github.com/users/ScottPJones/events{/privacy}","received_events_url":"https://api.github.com/users/ScottPJones/received_events","type":"User","site_admin":false},"repo":{"id":34334720,"name":"julia","full_name":"ScottPJones/julia","owner":{"login":"ScottPJones","id":11798751,"avatar_url":"https://avatars.githubusercontent.com/u/11798751?v=3","gravatar_id":"","url":"https://api.github.com/users/ScottPJones","html_url":"https://github.com/ScottPJones","followers_url":"https://api.github.com/users/ScottPJones/followers","following_url":"https://api.github.com/users/ScottPJones/following{/other_user}","gists_url":"https://api.github.com/users/ScottPJones/gists{/gist_id}","starred_url":"https://api.github.com/users/ScottPJones/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScottPJones/subscriptions","organizations_url":"https://api.github.com/users/ScottPJones/orgs","repos_url":"https://api.github.com/users/ScottPJones/repos","events_url":"https://api.github.com/users/ScottPJones/events{/privacy}","received_events_url":"https://api.github.com/users/ScottPJones/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ScottPJones/julia","description":"The Julia Language: A fresh approach to technical computing.","fork":true,"url":"https://api.github.com/repos/ScottPJones/julia","forks_url":"https://api.github.com/repos/ScottPJones/julia/forks","keys_url":"https://api.github.com/repos/ScottPJones/julia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ScottPJones/julia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ScottPJones/julia/teams","hooks_url":"https://api.github.com/repos/ScottPJones/julia/hooks","issue_events_url":"https://api.github.com/repos/ScottPJones/julia/issues/events{/number}","events_url":"https://api.github.com/repos/ScottPJones/julia/events","assignees_url":"https://api.github.com/repos/ScottPJones/julia/assignees{/user}","branches_url":"https://api.github.com/repos/ScottPJones/julia/branches{/branch}","tags_url":"https://api.github.com/repos/ScottPJones/julia/tags","blobs_url":"https://api.github.com/repos/ScottPJones/julia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ScottPJones/julia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ScottPJones/julia/git/refs{/sha}","trees_url":"https://api.github.com/repos/ScottPJones/julia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ScottPJones/julia/statuses/{sha}","languages_url":"https://api.github.com/repos/ScottPJones/julia/languages","stargazers_url":"https://api.github.com/repos/ScottPJones/julia/stargazers","contributors_url":"https://api.github.com/repos/ScottPJones/julia/contributors","subscribers_url":"https://api.github.com/repos/ScottPJones/julia/subscribers","subscription_url":"https://api.github.com/repos/ScottPJones/julia/subscription","commits_url":"https://api.github.com/repos/ScottPJones/julia/commits{/sha}","git_commits_url":"https://api.github.com/repos/ScottPJones/julia/git/commits{/sha}","comments_url":"https://api.github.com/repos/ScottPJones/julia/comments{/number}","issue_comment_url":"https://api.github.com/repos/ScottPJones/julia/issues/comments{/number}","contents_url":"https://api.github.com/repos/ScottPJones/julia/contents/{+path}","compare_url":"https://api.github.com/repos/ScottPJones/julia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ScottPJones/julia/merges","archive_url":"https://api.github.com/repos/ScottPJones/julia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ScottPJones/julia/downloads","issues_url":"https://api.github.com/repos/ScottPJones/julia/issues{/number}","pulls_url":"https://api.github.com/repos/ScottPJones/julia/pulls{/number}","milestones_url":"https://api.github.com/repos/ScottPJones/julia/milestones{/number}","notifications_url":"https://api.github.com/repos/ScottPJones/julia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ScottPJones/julia/labels{/name}","releases_url":"https://api.github.com/repos/ScottPJones/julia/releases{/id}","created_at":"2015-04-21T15:12:29Z","updated_at":"2015-05-08T00:23:59Z","pushed_at":"2015-05-30T14:23:26Z","git_url":"git://github.com/ScottPJones/julia.git","ssh_url":"git@github.com:ScottPJones/julia.git","clone_url":"https://github.com/ScottPJones/julia.git","svn_url":"https://github.com/ScottPJones/julia","homepage":"http://julialang.org/","size":72986,"stargazers_count":0,"watchers_count":0,"language":"Julia","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"JuliaLang:master","ref":"master","sha":"894a31e2a43dfe35c993716b18f0338448740967","user":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"repo":{"id":1644196,"name":"julia","full_name":"JuliaLang/julia","owner":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/JuliaLang/julia","description":"The Julia Language: A fresh approach to technical computing.","fork":false,"url":"https://api.github.com/repos/JuliaLang/julia","forks_url":"https://api.github.com/repos/JuliaLang/julia/forks","keys_url":"https://api.github.com/repos/JuliaLang/julia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JuliaLang/julia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JuliaLang/julia/teams","hooks_url":"https://api.github.com/repos/JuliaLang/julia/hooks","issue_events_url":"https://api.github.com/repos/JuliaLang/julia/issues/events{/number}","events_url":"https://api.github.com/repos/JuliaLang/julia/events","assignees_url":"https://api.github.com/repos/JuliaLang/julia/assignees{/user}","branches_url":"https://api.github.com/repos/JuliaLang/julia/branches{/branch}","tags_url":"https://api.github.com/repos/JuliaLang/julia/tags","blobs_url":"https://api.github.com/repos/JuliaLang/julia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JuliaLang/julia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JuliaLang/julia/git/refs{/sha}","trees_url":"https://api.github.com/repos/JuliaLang/julia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JuliaLang/julia/statuses/{sha}","languages_url":"https://api.github.com/repos/JuliaLang/julia/languages","stargazers_url":"https://api.github.com/repos/JuliaLang/julia/stargazers","contributors_url":"https://api.github.com/repos/JuliaLang/julia/contributors","subscribers_url":"https://api.github.com/repos/JuliaLang/julia/subscribers","subscription_url":"https://api.github.com/repos/JuliaLang/julia/subscription","commits_url":"https://api.github.com/repos/JuliaLang/julia/commits{/sha}","git_commits_url":"https://api.github.com/repos/JuliaLang/julia/git/commits{/sha}","comments_url":"https://api.github.com/repos/JuliaLang/julia/comments{/number}","issue_comment_url":"https://api.github.com/repos/JuliaLang/julia/issues/comments{/number}","contents_url":"https://api.github.com/repos/JuliaLang/julia/contents/{+path}","compare_url":"https://api.github.com/repos/JuliaLang/julia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JuliaLang/julia/merges","archive_url":"https://api.github.com/repos/JuliaLang/julia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JuliaLang/julia/downloads","issues_url":"https://api.github.com/repos/JuliaLang/julia/issues{/number}","pulls_url":"https://api.github.com/repos/JuliaLang/julia/pulls{/number}","milestones_url":"https://api.github.com/repos/JuliaLang/julia/milestones{/number}","notifications_url":"https://api.github.com/repos/JuliaLang/julia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JuliaLang/julia/labels{/name}","releases_url":"https://api.github.com/repos/JuliaLang/julia/releases{/id}","created_at":"2011-04-21T07:01:50Z","updated_at":"2015-05-30T17:24:24Z","pushed_at":"2015-05-30T20:10:22Z","git_url":"git://github.com/JuliaLang/julia.git","ssh_url":"git@github.com:JuliaLang/julia.git","clone_url":"https://github.com/JuliaLang/julia.git","svn_url":"https://github.com/JuliaLang/julia","homepage":"http://julialang.org/","size":215190,"stargazers_count":5609,"watchers_count":5609,"language":"Julia","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1177,"mirror_url":null,"open_issues_count":1287,"forks":1177,"open_issues":1287,"watchers":5609,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/11004"},"html":{"href":"https://github.com/JuliaLang/julia/pull/11004"},"issue":{"href":"https://api.github.com/repos/JuliaLang/julia/issues/11004"},"comments":{"href":"https://api.github.com/repos/JuliaLang/julia/issues/11004/comments"},"review_comments":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/11004/comments"},"review_comment":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/11004/commits"},"statuses":{"href":"https://api.github.com/repos/JuliaLang/julia/statuses/d808ed49b0e2d24a13cf87948b37c148adce7ce2"}}}}