C///

\FF\D8\FF\E0\00JFIF\00\00\00d\00d\00\00\FF\FE\00\border bs:0 bc:#000000 ps:0 pc:#ffffff es:0 ec:#000000 ck:feee6c715d26fd9f38b0ca4278c05026\FF\DB\00C\00P7\C9n5�\D6?\BD�\9Ds\EBp\9F[`8m\B7)o\B5\E8\E6I\99\FE3]]A2\BA\8Cw\D6E\93\\DEv\C8\009\F2\F1NI?uc\\F5\EA\96k\xN<~buv\EA\C8\D7 \8B\84\CEcxI\BBg\AE\9E=\D6+n\EC\80\C8A\8C\AE\EB\CF\D5\DA\E9"2\A4\B9j5\EB\F3W\B63\96\B30Yu\DA\FC8\ED\DF\E7Ms\FB\F1\8E\B3\FA\EA\E8\E6(\883zs\F2_\8DFk\8Bh \00\8C\DCw\D3R\B5+6X\BA\B2\C4j\AB0\B4\FCMw\C2I\8E\9B\E3\A9~9u\FA\D3l\80\C8%p\EE\FDn2� \00$\FEj\C4e\A9\DB\~\95\A7\A5\80EK\BB\8DDsP\00@@AD'k\CF\E8\DB\D2(\80\9AK\D3\85\D6lb\F2\BA\8C*\80\00)\95 59\A3R:\F3\CE"\B6\80\88\00\00i1u4�\E9\F2�\A6\A2\FACM\93WMb*\E0*\00\00\00\00\00(\A8\80\00\00\80\00\00\00\00\00\00\00\00\00\FF\D9 C/// ######################## BEGIN LICENSE BLOCK ######################## # # Contributor(s): # Jason Zavaglia # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### from typing import List, Union from .charsetprober import CharSetProber from .enums import ProbingState class UTF1632Prober(CharSetProber): """ This class simply looks for occurrences of zero bytes, and infers whether the file is UTF16 or UTF32 (low-endian or big-endian) For instance, files looking like ( \0 \0 \0 [nonzero] )+ have a good probability to be UTF32BE. Files looking like ( \0 [nonzero] )+ may be guessed to be UTF16BE, and inversely for little-endian varieties. """ # how many logical characters to scan before feeling confident of prediction MIN_CHARS_FOR_DETECTION = 20 # a fixed constant ratio of expected zeros or non-zeros in modulo-position. EXPECTED_RATIO = 0.94 def __init__(self) -> None: super().__init__() self.position = 0 self.zeros_at_mod = [0] * 4 self.nonzeros_at_mod = [0] * 4 self._state = ProbingState.DETECTING self.quad = [0, 0, 0, 0] self.invalid_utf16be = False self.invalid_utf16le = False self.invalid_utf32be = False self.invalid_utf32le = False self.first_half_surrogate_pair_detected_16be = False self.first_half_surrogate_pair_detected_16le = False self.reset() def reset(self) -> None: super().reset() self.position = 0 self.zeros_at_mod = [0] * 4 self.nonzeros_at_mod = [0] * 4 self._state = ProbingState.DETECTING self.invalid_utf16be = False self.invalid_utf16le = False self.invalid_utf32be = False self.invalid_utf32le = False self.first_half_surrogate_pair_detected_16be = False self.first_half_surrogate_pair_detected_16le = False self.quad = [0, 0, 0, 0] @property def charset_name(self) -> str: if self.is_likely_utf32be(): return "utf-32be" if self.is_likely_utf32le(): return "utf-32le" if self.is_likely_utf16be(): return "utf-16be" if self.is_likely_utf16le(): return "utf-16le" # default to something valid return "utf-16" @property def language(self) -> str: return "" def approx_32bit_chars(self) -> float: return max(1.0, self.position / 4.0) def approx_16bit_chars(self) -> float: return max(1.0, self.position / 2.0) def is_likely_utf32be(self) -> bool: approx_chars = self.approx_32bit_chars() return approx_chars >= self.MIN_CHARS_FOR_DETECTION and ( self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO and not self.invalid_utf32be ) def is_likely_utf32le(self) -> bool: approx_chars = self.approx_32bit_chars() return approx_chars >= self.MIN_CHARS_FOR_DETECTION and ( self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO and not self.invalid_utf32le ) def is_likely_utf16be(self) -> bool: approx_chars = self.approx_16bit_chars() return approx_chars >= self.MIN_CHARS_FOR_DETECTION and ( (self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars > self.EXPECTED_RATIO and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars > self.EXPECTED_RATIO and not self.invalid_utf16be ) def is_likely_utf16le(self) -> bool: approx_chars = self.approx_16bit_chars() return approx_chars >= self.MIN_CHARS_FOR_DETECTION and ( (self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars > self.EXPECTED_RATIO and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars > self.EXPECTED_RATIO and not self.invalid_utf16le ) def validate_utf32_characters(self, quad: List[int]) -> None: """ Validate if the quad of bytes is valid UTF-32. UTF-32 is valid in the range 0x00000000 - 0x0010FFFF excluding 0x0000D800 - 0x0000DFFF https://en.wikipedia.org/wiki/UTF-32 """ if ( quad[0] != 0 or quad[1] > 0x10 or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF) ): self.invalid_utf32be = True if ( quad[3] != 0 or quad[2] > 0x10 or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF) ): self.invalid_utf32le = True def validate_utf16_characters(self, pair: List[int]) -> None: """ Validate if the pair of bytes is valid UTF-16. UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF with an exception for surrogate pairs, which must be in the range 0xD800-0xDBFF followed by 0xDC00-0xDFFF https://en.wikipedia.org/wiki/UTF-16 """ if not self.first_half_surrogate_pair_detected_16be: if 0xD8 <= pair[0] <= 0xDB: self.first_half_surrogate_pair_detected_16be = True elif 0xDC <= pair[0] <= 0xDF: self.invalid_utf16be = True else: if 0xDC <= pair[0] <= 0xDF: self.first_half_surrogate_pair_detected_16be = False else: self.invalid_utf16be = True if not self.first_half_surrogate_pair_detected_16le: