cotugno/jellyfin
2
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2024-11-20 04:18:51 -07:00
Code Issues Packages Projects Releases Wiki Activity
58d6778876
jellyfin/Emby.Common.Implementations/TextEncoding/UniversalDetector/Core/MBCSSM.cs
Luke Pulverenti 4d7d8961b4 update encoding detection
2017-04-18 01:53:39 -04:00

641 lines
34 KiB
C#
Raw Blame History

/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is Mozilla Universal charset detector code.
*
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 2001
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* Shy Shalom <shooshX@gmail.com>
* Rudi Pettazzi <rudi.pettazzi@gmail.com> (C# port)
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
namespace UniversalDetector.Core
{
public class UTF8SMModel : SMModel
{
private readonly static int[] UTF8_cls = {
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 40 - 47
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 48 - 4f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 50 - 57
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 58 - 5f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 60 - 67
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 68 - 6f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 70 - 77
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 78 - 7f
BitPackage.Pack4bits(2,2,2,2,3,3,3,3), // 80 - 87
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 88 - 8f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 90 - 97
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 98 - 9f
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // a0 - a7
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // a8 - af
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // b0 - b7
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // b8 - bf
BitPackage.Pack4bits(0,0,6,6,6,6,6,6), // c0 - c7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c8 - cf
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d0 - d7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d8 - df
BitPackage.Pack4bits(7,8,8,8,8,8,8,8), // e0 - e7
BitPackage.Pack4bits(8,8,8,8,8,9,8,8), // e8 - ef
BitPackage.Pack4bits(10,11,11,11,11,11,11,11), // f0 - f7
BitPackage.Pack4bits(12,13,13,13,14,15,0,0) // f8 - ff
};
private readonly static int[] UTF8_st = {
BitPackage.Pack4bits(ERROR,START,ERROR,ERROR,ERROR,ERROR, 12, 10),//00-07
BitPackage.Pack4bits( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//10-17
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f
BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//20-27
BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//28-2f
BitPackage.Pack4bits(ERROR,ERROR, 5, 5, 5, 5,ERROR,ERROR),//30-37
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//38-3f
BitPackage.Pack4bits(ERROR,ERROR,ERROR, 5, 5, 5,ERROR,ERROR),//40-47
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//48-4f
BitPackage.Pack4bits(ERROR,ERROR, 7, 7, 7, 7,ERROR,ERROR),//50-57
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//58-5f
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR, 7, 7,ERROR,ERROR),//60-67
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//68-6f
BitPackage.Pack4bits(ERROR,ERROR, 9, 9, 9, 9,ERROR,ERROR),//70-77
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//78-7f
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR, 9,ERROR,ERROR),//80-87
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//88-8f
BitPackage.Pack4bits(ERROR,ERROR, 12, 12, 12, 12,ERROR,ERROR),//90-97
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//98-9f
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR, 12,ERROR,ERROR),//a0-a7
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//a8-af
BitPackage.Pack4bits(ERROR,ERROR, 12, 12, 12,ERROR,ERROR,ERROR),//b0-b7
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//b8-bf
BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,ERROR,ERROR),//c0-c7
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR) //c8-cf
};
private readonly static int[] UTF8CharLenTable =
{0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6 };
public UTF8SMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UTF8_cls),
16,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UTF8_st),
UTF8CharLenTable, "UTF-8")
{
}
}
public class GB18030SMModel : SMModel
{
private readonly static int[] GB18030_cls = {
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 30 - 37
BitPackage.Pack4bits(3,3,1,1,1,1,1,1), // 38 - 3f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77
BitPackage.Pack4bits(2,2,2,2,2,2,2,4), // 78 - 7f
BitPackage.Pack4bits(5,6,6,6,6,6,6,6), // 80 - 87
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 88 - 8f
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 90 - 97
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 98 - 9f
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // a0 - a7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // a8 - af
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // b0 - b7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // b8 - bf
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c0 - c7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c8 - cf
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d0 - d7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d8 - df
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // e0 - e7
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // e8 - ef
BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // f0 - f7
BitPackage.Pack4bits(6,6,6,6,6,6,6,0) // f8 - ff
};
private readonly static int[] GB18030_st = {
BitPackage.Pack4bits(ERROR,START,START,START,START,START, 3,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START),//10-17
BitPackage.Pack4bits( 4,ERROR,START,START,ERROR,ERROR,ERROR,ERROR),//18-1f
BitPackage.Pack4bits(ERROR,ERROR, 5,ERROR,ERROR,ERROR,ITSME,ERROR),//20-27
BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,START,START) //28-2f
};
// To be accurate, the length of class 6 can be either 2 or 4.
// But it is not necessary to discriminate between the two since
// it is used for frequency analysis only, and we are validating
// each code range there as well. So it is safe to set it to be
// 2 here.
private readonly static int[] GB18030CharLenTable = {0, 1, 1, 1, 1, 1, 2};
public GB18030SMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, GB18030_cls),
7,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, GB18030_st),
GB18030CharLenTable, "GB18030")
{
}
}
public class BIG5SMModel : SMModel
{
private readonly static int[] BIG5_cls = {
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77
BitPackage.Pack4bits(2,2,2,2,2,2,2,1), // 78 - 7f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 80 - 87
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 88 - 8f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 90 - 97
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 98 - 9f
BitPackage.Pack4bits(4,3,3,3,3,3,3,3), // a0 - a7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // a8 - af
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // b0 - b7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // b8 - bf
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c0 - c7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c8 - cf
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d0 - d7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d8 - df
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e8 - ef
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // f0 - f7
BitPackage.Pack4bits(3,3,3,3,3,3,3,0) // f8 - ff
};
private readonly static int[] BIG5_st = {
BitPackage.Pack4bits(ERROR,START,START, 3,ERROR,ERROR,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ITSME,ITSME,ITSME,ITSME,ITSME,ERROR),//08-0f
BitPackage.Pack4bits(ERROR,START,START,START,START,START,START,START) //10-17
};
private readonly static int[] BIG5CharLenTable = {0, 1, 1, 2, 0};
public BIG5SMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, BIG5_cls),
5,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, BIG5_st),
BIG5CharLenTable, "Big5")
{
}
}
public class EUCJPSMModel : SMModel
{
private readonly static int[] EUCJP_cls = {
//BitPacket.Pack4bits(5,4,4,4,4,4,4,4), // 00 - 07
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 00 - 07
BitPackage.Pack4bits(4,4,4,4,4,4,5,5), // 08 - 0f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 10 - 17
BitPackage.Pack4bits(4,4,4,5,4,4,4,4), // 18 - 1f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 20 - 27
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 28 - 2f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 30 - 37
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 38 - 3f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 40 - 47
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 48 - 4f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 50 - 57
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 58 - 5f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 60 - 67
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 68 - 6f
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 70 - 77
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 78 - 7f
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 80 - 87
BitPackage.Pack4bits(5,5,5,5,5,5,1,3), // 88 - 8f
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 90 - 97
BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 98 - 9f
BitPackage.Pack4bits(5,2,2,2,2,2,2,2), // a0 - a7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a8 - af
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c8 - cf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7
BitPackage.Pack4bits(0,0,0,0,0,0,0,5) // f8 - ff
};
private readonly static int[] EUCJP_st = {
BitPackage.Pack4bits( 3, 4, 3, 5,START,ERROR,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME,START,ERROR,START,ERROR,ERROR,ERROR),//10-17
BitPackage.Pack4bits(ERROR,ERROR,START,ERROR,ERROR,ERROR, 3,ERROR),//18-1f
BitPackage.Pack4bits( 3,ERROR,ERROR,ERROR,START,START,START,START) //20-27
};
private readonly static int[] EUCJPCharLenTable = { 2, 2, 2, 3, 1, 0 };
public EUCJPSMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCJP_cls),
6,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCJP_st),
EUCJPCharLenTable, "EUC-JP")
{
}
}
public class EUCKRSMModel : SMModel
{
private readonly static int[] EUCKR_cls = {
//BitPacket.Pack4bits(0,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 40 - 47
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 48 - 4f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 50 - 57
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 58 - 5f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 60 - 67
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 68 - 6f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 70 - 77
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 78 - 7f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f
BitPackage.Pack4bits(0,2,2,2,2,2,2,2), // a0 - a7
BitPackage.Pack4bits(2,2,2,2,2,3,3,3), // a8 - af
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7
BitPackage.Pack4bits(2,3,2,2,2,2,2,2), // c8 - cf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // e0 - e7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // e8 - ef
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // f0 - f7
BitPackage.Pack4bits(2,2,2,2,2,2,2,0) // f8 - ff
};
private readonly static int[] EUCKR_st = {
BitPackage.Pack4bits(ERROR,START, 3,ERROR,ERROR,ERROR,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START,START) //08-0f
};
private readonly static int[] EUCKRCharLenTable = { 0, 1, 2, 0 };
public EUCKRSMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCKR_cls),
4,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCKR_st),
EUCKRCharLenTable, "EUC-KR")
{
}
}
public class EUCTWSMModel : SMModel
{
private readonly static int[] EUCTW_cls = {
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 00 - 07
BitPackage.Pack4bits(2,2,2,2,2,2,0,0), // 08 - 0f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 10 - 17
BitPackage.Pack4bits(2,2,2,0,2,2,2,2), // 18 - 1f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 20 - 27
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 28 - 2f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 30 - 37
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 38 - 3f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 78 - 7f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87
BitPackage.Pack4bits(0,0,0,0,0,0,6,0), // 88 - 8f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f
BitPackage.Pack4bits(0,3,4,4,4,4,4,4), // a0 - a7
BitPackage.Pack4bits(5,5,1,1,1,1,1,1), // a8 - af
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // b0 - b7
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // b8 - bf
BitPackage.Pack4bits(1,1,3,1,3,3,3,3), // c0 - c7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c8 - cf
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d0 - d7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d8 - df
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e8 - ef
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // f0 - f7
BitPackage.Pack4bits(3,3,3,3,3,3,3,0) // f8 - ff
};
private readonly static int[] EUCTW_st = {
BitPackage.Pack4bits(ERROR,ERROR,START, 3, 3, 3, 4,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,START,ERROR),//10-17
BitPackage.Pack4bits(START,START,START,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f
BitPackage.Pack4bits( 5,ERROR,ERROR,ERROR,START,ERROR,START,START),//20-27
BitPackage.Pack4bits(START,ERROR,START,START,START,START,START,START) //28-2f
};
private readonly static int[] EUCTWCharLenTable = { 0, 0, 1, 2, 2, 2, 3 };
public EUCTWSMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCTW_cls),
7,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, EUCTW_st),
EUCTWCharLenTable, "EUC-TW")
{
}
}
public class SJISSMModel : SMModel
{
private readonly static int[] SJIS_cls = {
//BitPacket.Pack4bits(0,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17
BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77
BitPackage.Pack4bits(2,2,2,2,2,2,2,1), // 78 - 7f
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 80 - 87
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 88 - 8f
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 90 - 97
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 98 - 9f
//0xa0 is illegal in sjis encoding, but some pages does
//contain such byte. We need to be more error forgiven.
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a0 - a7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a8 - af
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c8 - cf
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df
BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7
BitPackage.Pack4bits(3,3,3,3,3,4,4,4), // e8 - ef
BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // f0 - f7
BitPackage.Pack4bits(4,4,4,4,4,0,0,0) // f8 - ff
};
private readonly static int[] SJIS_st = {
BitPackage.Pack4bits(ERROR,START,START, 3,ERROR,ERROR,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME,ERROR,ERROR,START,START,START,START) //10-17
};
private readonly static int[] SJISCharLenTable = { 0, 1, 1, 2, 0, 0 };
public SJISSMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, SJIS_cls),
6,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, SJIS_st),
SJISCharLenTable, "Shift_JIS")
{
}
}
public class UCS2BESMModel : SMModel
{
private readonly static int[] UCS2BE_cls = {
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 00 - 07
BitPackage.Pack4bits(0,0,1,0,0,2,0,0), // 08 - 0f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 10 - 17
BitPackage.Pack4bits(0,0,0,3,0,0,0,0), // 18 - 1f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 20 - 27
BitPackage.Pack4bits(0,3,3,3,3,3,0,0), // 28 - 2f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 30 - 37
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 38 - 3f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 40 - 47
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 48 - 4f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 50 - 57
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 58 - 5f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 60 - 67
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 68 - 6f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 70 - 77
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 78 - 7f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a0 - a7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a8 - af
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b0 - b7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b8 - bf
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c0 - c7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c8 - cf
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d0 - d7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d8 - df
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7
BitPackage.Pack4bits(0,0,0,0,0,0,4,5) // f8 - ff
};
private readonly static int[] UCS2BE_st = {
BitPackage.Pack4bits( 5, 7, 7,ERROR, 4, 3,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME, 6, 6, 6, 6,ERROR,ERROR),//10-17
BitPackage.Pack4bits( 6, 6, 6, 6, 6,ITSME, 6, 6),//18-1f
BitPackage.Pack4bits( 6, 6, 6, 6, 5, 7, 7,ERROR),//20-27
BitPackage.Pack4bits( 5, 8, 6, 6,ERROR, 6, 6, 6),//28-2f
BitPackage.Pack4bits( 6, 6, 6, 6,ERROR,ERROR,START,START) //30-37
};
private readonly static int[] UCS2BECharLenTable = { 2, 2, 2, 0, 2, 2 };
public UCS2BESMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UCS2BE_cls),
6,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UCS2BE_st),
UCS2BECharLenTable, "UTF-16BE")
{
}
}
public class UCS2LESMModel : SMModel
{
private readonly static int[] UCS2LE_cls = {
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 00 - 07
BitPackage.Pack4bits(0,0,1,0,0,2,0,0), // 08 - 0f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 10 - 17
BitPackage.Pack4bits(0,0,0,3,0,0,0,0), // 18 - 1f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 20 - 27
BitPackage.Pack4bits(0,3,3,3,3,3,0,0), // 28 - 2f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 30 - 37
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 38 - 3f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 40 - 47
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 48 - 4f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 50 - 57
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 58 - 5f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 60 - 67
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 68 - 6f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 70 - 77
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 78 - 7f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a0 - a7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a8 - af
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b0 - b7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b8 - bf
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c0 - c7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c8 - cf
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d0 - d7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d8 - df
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef
BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7
BitPackage.Pack4bits(0,0,0,0,0,0,4,5) // f8 - ff
};
private readonly static int[] UCS2LE_st = {
BitPackage.Pack4bits( 6, 6, 7, 6, 4, 3,ERROR,ERROR),//00-07
BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f
BitPackage.Pack4bits(ITSME,ITSME, 5, 5, 5,ERROR,ITSME,ERROR),//10-17
BitPackage.Pack4bits( 5, 5, 5,ERROR, 5,ERROR, 6, 6),//18-1f
BitPackage.Pack4bits( 7, 6, 8, 8, 5, 5, 5,ERROR),//20-27
BitPackage.Pack4bits( 5, 5, 5,ERROR,ERROR,ERROR, 5, 5),//28-2f
BitPackage.Pack4bits( 5, 5, 5,ERROR, 5,ERROR,START,START) //30-37
};
private readonly static int[] UCS2LECharLenTable = { 2, 2, 2, 2, 2, 2 };
public UCS2LESMModel() : base(
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UCS2LE_cls),
6,
new BitPackage(BitPackage.INDEX_SHIFT_4BITS,
BitPackage.SHIFT_MASK_4BITS,
BitPackage.BIT_SHIFT_4BITS,
BitPackage.UNIT_MASK_4BITS, UCS2LE_st),
UCS2LECharLenTable, "UTF-16LE")
{
}
}
}
Reference in New Issue View Git Blame Copy Permalink