Codepage auto detect & yj_2 fix

common.h

@@ -229,6 +229,8 @@ typedef const WCHAR        *LPCWSTR;
 #define WIDETEXT(quote) __WIDETEXT(quote)
 
 typedef enum tagCODEPAGE {
+	CP_UNKNOWN = -1,
+	CP_MIN = 0,
 	CP_BIG5 = 0,
 	CP_GBK = 1,
 	CP_SHIFTJIS = 2,

global.c

@@ -25,17 +25,24 @@
 
 LPGLOBALVARS gpGlobals = NULL;
 
+#if SDL_BYTEORDER == SDL_LIL_ENDIAN
+#define DO_BYTESWAP(buf, size)
+#else
 #define DO_BYTESWAP(buf, size)                                   \
-   for (i = 0; i < (size) / 2; i++)                              \
-   {                                                             \
-      ((LPWORD)(buf))[i] = SWAP16(((LPWORD)(buf))[i]);           \
-   }
+   do {                                                          \
+      int i;                                                     \
+      for (i = 0; i < (size) / 2; i++)                           \
+      {                                                          \
+         ((LPWORD)(buf))[i] = SWAP16(((LPWORD)(buf))[i]);        \
+      }                                                          \
+   } while(0)
+#endif
 
 #define LOAD_DATA(buf, size, chunknum, fp)                       \
-   {                                                             \
+   do {                                                          \
       PAL_MKFReadChunk((LPBYTE)(buf), (size), (chunknum), (fp)); \
       DO_BYTESWAP(buf, size);                                    \
-   }
+   } while(0)
 
 INT
 PAL_InitGlobals(
@@ -59,7 +66,7 @@ PAL_InitGlobals(
 {
 #ifdef PAL_UNICODE
    FILE     *fp;
-   CODEPAGE  iCodePage = CP_BIG5;	// Default for PAL DOS/WIN95
+   CODEPAGE  iCodePage = CP_UNKNOWN;
    DWORD     dwWordLength = 10;		// Default for PAL DOS/WIN95
    DWORD     dwExtraMagicDescLines = 0;	// Default for PAL DOS/WIN95
    DWORD     dwExtraItemDescLines = 0;	// Default for PAL DOS/WIN95
@@ -129,6 +136,48 @@ PAL_InitGlobals(
 
 	   UTIL_CloseFile(fp);
    }
+
+   // Codepage auto detect 
+   if (CP_UNKNOWN == iCodePage)
+   {
+	   // Try to convert the content of word.dat with different codepages,
+	   // and use the codepage with minimal inconvertible characters
+	   // Works fine currently with SC/TC/JP.
+	   if (fp = UTIL_OpenFile("word.dat"))
+	   {
+		   char *buf;
+		   long len;
+		   int i, j, c, m = INT_MAX, m_i;
+		   fseek(fp, 0, SEEK_END);
+		   len = ftell(fp);
+		   buf = (char *)malloc(len);
+		   fseek(fp, 0, SEEK_SET);
+		   fread(buf, 1, len, fp);
+		   UTIL_CloseFile(fp);
+		   for (i = CP_MIN; i < CP_MAX; i++)
+		   {
+			   int wlen = PAL_MultiByteToWideChar(buf, len, NULL, 0);
+			   WCHAR *wbuf = (WCHAR *)malloc(wlen * sizeof(WCHAR));
+			   PAL_MultiByteToWideChar(buf, len, wbuf, wlen);
+			   for (j = c = 0; j < wlen; j++)
+			   {
+				   c += (wbuf[j] == PAL_GetInvalidChar(i)) ? 1 : 0;
+			   }
+			   if (c < m)
+			   {
+				   m = c;
+				   m_i = i;
+			   }
+			   free(wbuf);
+		   }
+		   free(buf);
+		   iCodePage = m_i;
+	   }
+	   else
+	   {
+		   iCodePage = CP_BIG5;
+	   }
+   }
 #endif
 
    //
@@ -237,7 +286,6 @@ PAL_ReadGlobalGameData(
 --*/
 {
    const GAMEDATA    *p = &gpGlobals->g;
-   unsigned int       i;
 
    LOAD_DATA(p->lprgScriptEntry, p->nScriptEntry * sizeof(SCRIPTENTRY),
       4, gpGlobals->f.fpSSS);
@@ -422,7 +470,6 @@ PAL_LoadGame(
 {
    FILE                     *fp;
    PAL_LARGE SAVEDGAME       s;
-   UINT32                    i;
 
    //
    // Try to open the specified file

text.c

@@ -1383,4 +1383,18 @@ PAL_MultiByteToWideChar(
 
 	}
 }
+
+WCHAR
+PAL_GetInvalidChar(
+   CODEPAGE      iCodePage
+)
+{
+   switch(iCodePage)
+   {
+   case CP_BIG5:     return 0x3f;
+   case CP_GBK:      return 0x3f;
+   case CP_SHIFTJIS: return 0x30fb;
+   default:          return 0;
+   }
+}
 #endif

text.h

@@ -125,6 +125,11 @@ PAL_MultiByteToWideChar(
    LPWSTR        wcs,
    int           wcslength
 );
+
+WCHAR
+PAL_GetInvalidChar(
+   CODEPAGE      iCodePage
+);
 #endif
 
 #endif

util.c

@@ -469,11 +469,7 @@ UTIL_OpenFile(
 	  char *p = pBuf;
 	  while (*p)
 	  {
-		 if (*p >= 'a' && *p <= 'z')
-		 {
-			*p -= 'a' - 'A';
-		 }
-		 p++;
+         *p++ = toupper(*p);
 	  }
 
 	  fp = fopen(va("%s%s", PAL_PREFIX, pBuf), "rb");

video.c

@@ -604,7 +604,6 @@ VIDEO_SetPalette(
 --*/
 {
 #if SDL_VERSION_ATLEAST(2,0,0)
-   int            i;
    SDL_Palette   *palette = SDL_AllocPalette(256);
 
    if (palette == NULL)

yj1.c

@@ -244,39 +244,39 @@ typedef struct _TreeNode
 {
    unsigned short      weight;
    unsigned short      value;
-   struct _TreeNode*   parent;
-   struct _TreeNode*   left;
-   struct _TreeNode*   right;
+   struct _TreeNode   *parent;
+   struct _TreeNode   *left;
+   struct _TreeNode   *right;
 } TreeNode;
 
 typedef struct _Tree
 {
-   TreeNode*   node;
-   TreeNode**   list;
+   TreeNode    *node;
+   TreeNode   **list;
 } Tree;
 
 static unsigned char data1[0x100] =
 {
-0x3f,0x0b,0x17,0x03,0x2f,0x0a,0x16,0x00,0x2e,0x09,0x15,0x02,0x2d,0x01,0x08,0x00,
-0x3e,0x07,0x14,0x03,0x2c,0x06,0x13,0x00,0x2b,0x05,0x12,0x02,0x2a,0x01,0x04,0x00,
-0x3d,0x0b,0x11,0x03,0x29,0x0a,0x10,0x00,0x28,0x09,0x0f,0x02,0x27,0x01,0x08,0x00,
-0x3c,0x07,0x0e,0x03,0x26,0x06,0x0d,0x00,0x25,0x05,0x0c,0x02,0x24,0x01,0x04,0x00,
-0x3b,0x0b,0x17,0x03,0x23,0x0a,0x16,0x00,0x22,0x09,0x15,0x02,0x21,0x01,0x08,0x00,
-0x3a,0x07,0x14,0x03,0x20,0x06,0x13,0x00,0x1f,0x05,0x12,0x02,0x1e,0x01,0x04,0x00,
-0x39,0x0b,0x11,0x03,0x1d,0x0a,0x10,0x00,0x1c,0x09,0x0f,0x02,0x1b,0x01,0x08,0x00,
-0x38,0x07,0x0e,0x03,0x1a,0x06,0x0d,0x00,0x19,0x05,0x0c,0x02,0x18,0x01,0x04,0x00,
-0x37,0x0b,0x17,0x03,0x2f,0x0a,0x16,0x00,0x2e,0x09,0x15,0x02,0x2d,0x01,0x08,0x00,
-0x36,0x07,0x14,0x03,0x2c,0x06,0x13,0x00,0x2b,0x05,0x12,0x02,0x2a,0x01,0x04,0x00,
-0x35,0x0b,0x11,0x03,0x29,0x0a,0x10,0x00,0x28,0x09,0x0f,0x02,0x27,0x01,0x08,0x00,
-0x34,0x07,0x0e,0x03,0x26,0x06,0x0d,0x00,0x25,0x05,0x0c,0x02,0x24,0x01,0x04,0x00,
-0x33,0x0b,0x17,0x03,0x23,0x0a,0x16,0x00,0x22,0x09,0x15,0x02,0x21,0x01,0x08,0x00,
-0x32,0x07,0x14,0x03,0x20,0x06,0x13,0x00,0x1f,0x05,0x12,0x02,0x1e,0x01,0x04,0x00,
-0x31,0x0b,0x11,0x03,0x1d,0x0a,0x10,0x00,0x1c,0x09,0x0f,0x02,0x1b,0x01,0x08,0x00,
-0x30,0x07,0x0e,0x03,0x1a,0x06,0x0d,0x00,0x19,0x05,0x0c,0x02,0x18,0x01,0x04,0x00
+	0x3f, 0x0b, 0x17, 0x03, 0x2f, 0x0a, 0x16, 0x00, 0x2e, 0x09, 0x15, 0x02, 0x2d, 0x01, 0x08, 0x00,
+	0x3e, 0x07, 0x14, 0x03, 0x2c, 0x06, 0x13, 0x00, 0x2b, 0x05, 0x12, 0x02, 0x2a, 0x01, 0x04, 0x00,
+	0x3d, 0x0b, 0x11, 0x03, 0x29, 0x0a, 0x10, 0x00, 0x28, 0x09, 0x0f, 0x02, 0x27, 0x01, 0x08, 0x00,
+	0x3c, 0x07, 0x0e, 0x03, 0x26, 0x06, 0x0d, 0x00, 0x25, 0x05, 0x0c, 0x02, 0x24, 0x01, 0x04, 0x00,
+	0x3b, 0x0b, 0x17, 0x03, 0x23, 0x0a, 0x16, 0x00, 0x22, 0x09, 0x15, 0x02, 0x21, 0x01, 0x08, 0x00,
+	0x3a, 0x07, 0x14, 0x03, 0x20, 0x06, 0x13, 0x00, 0x1f, 0x05, 0x12, 0x02, 0x1e, 0x01, 0x04, 0x00,
+	0x39, 0x0b, 0x11, 0x03, 0x1d, 0x0a, 0x10, 0x00, 0x1c, 0x09, 0x0f, 0x02, 0x1b, 0x01, 0x08, 0x00,
+	0x38, 0x07, 0x0e, 0x03, 0x1a, 0x06, 0x0d, 0x00, 0x19, 0x05, 0x0c, 0x02, 0x18, 0x01, 0x04, 0x00,
+	0x37, 0x0b, 0x17, 0x03, 0x2f, 0x0a, 0x16, 0x00, 0x2e, 0x09, 0x15, 0x02, 0x2d, 0x01, 0x08, 0x00,
+	0x36, 0x07, 0x14, 0x03, 0x2c, 0x06, 0x13, 0x00, 0x2b, 0x05, 0x12, 0x02, 0x2a, 0x01, 0x04, 0x00,
+	0x35, 0x0b, 0x11, 0x03, 0x29, 0x0a, 0x10, 0x00, 0x28, 0x09, 0x0f, 0x02, 0x27, 0x01, 0x08, 0x00,
+	0x34, 0x07, 0x0e, 0x03, 0x26, 0x06, 0x0d, 0x00, 0x25, 0x05, 0x0c, 0x02, 0x24, 0x01, 0x04, 0x00,
+	0x33, 0x0b, 0x17, 0x03, 0x23, 0x0a, 0x16, 0x00, 0x22, 0x09, 0x15, 0x02, 0x21, 0x01, 0x08, 0x00,
+	0x32, 0x07, 0x14, 0x03, 0x20, 0x06, 0x13, 0x00, 0x1f, 0x05, 0x12, 0x02, 0x1e, 0x01, 0x04, 0x00,
+	0x31, 0x0b, 0x11, 0x03, 0x1d, 0x0a, 0x10, 0x00, 0x1c, 0x09, 0x0f, 0x02, 0x1b, 0x01, 0x08, 0x00,
+	0x30, 0x07, 0x0e, 0x03, 0x1a, 0x06, 0x0d, 0x00, 0x19, 0x05, 0x0c, 0x02, 0x18, 0x01, 0x04, 0x00
 };
 static unsigned char data2[0x10] =
 {
-0x08,0x05,0x06,0x04,0x07,0x05,0x06,0x03,0x07,0x05,0x06,0x04,0x07,0x04,0x05,0x03
+	0x08, 0x05, 0x06, 0x04, 0x07, 0x05, 0x06, 0x03, 0x07, 0x05, 0x06, 0x04, 0x07, 0x04, 0x05, 0x03
 };
 
 static void adjust_tree(Tree tree, unsigned short value)
@@ -351,68 +351,11 @@ static int build_tree(Tree *tree)
    return 1;
 }
 
-#pragma pack(1)
-typedef struct _BitField
+static int bt(const char* data, unsigned int pos)
 {
-   unsigned char   b0:   1;
-   unsigned char   b1:   1;
-   unsigned char   b2:   1;
-   unsigned char   b3:   1;
-   unsigned char   b4:   1;
-   unsigned char   b5:   1;
-   unsigned char   b6:   1;
-   unsigned char   b7:   1;
-} BitField;
-#pragma pack()
-
-static int bt(const void* data, unsigned int pos)
-{
-   BitField* bit = (BitField*)((unsigned char*)data + (pos >> 3));
-   switch(pos & 0x7)
-   {
-   case 0:   return bit->b0;
-   case 1:   return bit->b1;
-   case 2:   return bit->b2;
-   case 3:   return bit->b3;
-   case 4:   return bit->b4;
-   case 5:   return bit->b5;
-   case 6:   return bit->b6;
-   case 7:   return bit->b7;
-   }
-   return 0;
+   return (data[pos >> 3] & (unsigned char)(1 << (pos & 0x7))) >> (pos & 0x7);
 }
 
-static void bit(void* data, unsigned int pos, int set)
-{
-   BitField* bit = (BitField*)((unsigned char*)data + (pos >> 3));
-   switch(pos & 0x7)
-   {
-   case 0:
-      bit->b0 = set;
-      break;
-   case 1:
-      bit->b1 = set;
-      break;
-   case 2:
-      bit->b2 = set;
-      break;
-   case 3:
-      bit->b3 = set;
-      break;
-   case 4:
-      bit->b4 = set;
-      break;
-   case 5:
-      bit->b5 = set;
-      break;
-   case 6:
-      bit->b6 = set;
-      break;
-   case 7:
-      bit->b7 = set;
-      break;
-   }
-}
 
 INT
 Decompress(