Fossil

Changes On Branch fossil_utf8_to_filename
Login

Changes On Branch fossil_utf8_to_filename

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch fossil_utf8_to_filename Excluding Merge-Ins

This is equivalent to a diff from a98467b6 to cc3976fd

2013-02-21
11:29
Rename fossil_utf8_to_unicode to fossil_utf8_to_filename on UNIX, everywhere where filenames are involved. This enables fossil_utf8_to_unicode/fossil_unicode_to_utf8 to be re-implemented on UNIX to do what the name suggests. ... (check-in: 7eb5e231 user: jan.nijtmans tags: trunk)
2013-02-18
13:46
Fixed ticket [5df2715635b99bd46a] (check-in count mismatch). ... (check-in: b27c0d6d user: stephan tags: trunk)
10:03
New function fossil_utf8_to_filename, such that fossil_unicode_to_utf8/fossil_utf8_to_unicode/fossil_unicode_free are not used on UNIX/MAC any more: On UNIX those 3 functions were only no-ops, but this allows to re-implement then for real unicode <-> utf-8 conversions. There is an "#ifdef _WIN32" around those 3 functions and 2 more (fossil_mbcs_to_utf8/fossil_mbcs_free) in order to prove that they are not used any more on UNIX/MAC. ... (Closed-Leaf check-in: cc3976fd user: jan.nijtmans tags: fossil_utf8_to_filename)
08:30
merge trunk ... (check-in: fdd51b61 user: jan.nijtmans tags: ticket-d17d6e5b17)
2013-02-17
21:37
merge trunk ... (check-in: fdf9050c user: jan.nijtmans tags: improve_commit_warning)
14:47
More simplification in UTF-16 bom detection ... (Closed-Leaf check-in: 1e70f211 user: jan.nijtmans tags: utf16Bom)
14:43
Remove two unused variables ... (check-in: a98467b6 user: jan.nijtmans tags: trunk)
2013-02-16
14:12
Limit the complexity of the diff display on check-in information pages. ... (check-in: 4f95ea8c user: drh tags: trunk)

Changes to src/file.c.

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  if( isWd && g.allowSymlinks ){
    return lstat(zFilename, buf);
  }else{
    return stat(zFilename, buf);
  }
#else
  int rc = 0;
  wchar_t *zMbcs = fossil_utf8_to_unicode(zFilename);
  rc = _wstati64(zMbcs, buf);
  fossil_unicode_free(zMbcs);
  return rc;
#endif
}

/*
** Fill in the fileStat variable for the file named zFilename.
** If zFilename==0, then use the previous value of fileStat if







|

|







70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  if( isWd && g.allowSymlinks ){
    return lstat(zFilename, buf);
  }else{
    return stat(zFilename, buf);
  }
#else
  int rc = 0;
  wchar_t *zMbcs = fossil_utf8_to_filename(zFilename);
  rc = _wstati64(zMbcs, buf);
  fossil_filename_free(zMbcs);
  return rc;
#endif
}

/*
** Fill in the fileStat variable for the file named zFilename.
** If zFilename==0, then use the previous value of fileStat if
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317


/*
** Wrapper around the access() system call.
*/
int file_access(const char *zFilename, int flags){
#ifdef _WIN32
  wchar_t *zMbcs = fossil_utf8_to_unicode(zFilename);
  int rc = _waccess(zMbcs, flags);
  fossil_unicode_free(zMbcs);
#else
  int rc = access(zFilename, flags);
#endif
  return rc;
}

/*







|

|







301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317


/*
** Wrapper around the access() system call.
*/
int file_access(const char *zFilename, int flags){
#ifdef _WIN32
  wchar_t *zMbcs = fossil_utf8_to_filename(zFilename);
  int rc = _waccess(zMbcs, flags);
  fossil_filename_free(zMbcs);
#else
  int rc = access(zFilename, flags);
#endif
  return rc;
}

/*
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
  struct timeval tv[2];
  memset(tv, 0, sizeof(tv[0])*2);
  tv[0].tv_sec = newMTime;
  tv[1].tv_sec = newMTime;
  utimes(zFilename, tv);
#else
  struct _utimbuf tb;
  wchar_t *zMbcs = fossil_utf8_to_unicode(zFilename);
  tb.actime = newMTime;
  tb.modtime = newMTime;
  _wutime(zMbcs, &tb);
  fossil_unicode_free(zMbcs);
#endif
}

/*
** COMMAND: test-set-mtime
**
** Usage: %fossil test-set-mtime FILENAME DATE/TIME







|



|







403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
  struct timeval tv[2];
  memset(tv, 0, sizeof(tv[0])*2);
  tv[0].tv_sec = newMTime;
  tv[1].tv_sec = newMTime;
  utimes(zFilename, tv);
#else
  struct _utimbuf tb;
  wchar_t *zMbcs = fossil_utf8_to_filename(zFilename);
  tb.actime = newMTime;
  tb.modtime = newMTime;
  _wutime(zMbcs, &tb);
  fossil_filename_free(zMbcs);
#endif
}

/*
** COMMAND: test-set-mtime
**
** Usage: %fossil test-set-mtime FILENAME DATE/TIME
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
}

/*
** Delete a file.
*/
void file_delete(const char *zFilename){
#ifdef _WIN32
  wchar_t *z = fossil_utf8_to_unicode(zFilename);
  _wunlink(z);
  fossil_unicode_free(z);
#else
  unlink(zFilename);
#endif
}

/*
** Create the directory named in the argument, if it does not already







|

|







439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
}

/*
** Delete a file.
*/
void file_delete(const char *zFilename){
#ifdef _WIN32
  wchar_t *z = fossil_utf8_to_filename(zFilename);
  _wunlink(z);
  fossil_filename_free(z);
#else
  unlink(zFilename);
#endif
}

/*
** Create the directory named in the argument, if it does not already
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
  if( rc==2 ){
    if( !forceFlag ) return 1;
    file_delete(zName);
  }
  if( rc!=1 ){
#if defined(_WIN32)
    int rc;
    wchar_t *zMbcs = fossil_utf8_to_unicode(zName);
    rc = _wmkdir(zMbcs);
    fossil_unicode_free(zMbcs);
    return rc;
#else
    return mkdir(zName, 0755);
#endif
  }
  return 0;
}







|

|







463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
  if( rc==2 ){
    if( !forceFlag ) return 1;
    file_delete(zName);
  }
  if( rc!=1 ){
#if defined(_WIN32)
    int rc;
    wchar_t *zMbcs = fossil_utf8_to_filename(zName);
    rc = _wmkdir(zMbcs);
    fossil_filename_free(zMbcs);
    return rc;
#else
    return mkdir(zName, 0755);
#endif
  }
  return 0;
}
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150

/*
** Return the value of an environment variable as UTF8.
** Use fossil_filename_free() to release resources.
*/
char *fossil_getenv(const char *zName){
#ifdef _WIN32
  wchar_t *uName = fossil_utf8_to_unicode(zName);
  void *zValue = _wgetenv(uName);
  fossil_unicode_free(uName);
#else
  char *zValue = getenv(zName);
#endif
  if( zValue ) zValue = fossil_filename_to_utf8(zValue);
  return zValue;
}

/*
** Like fopen() but always takes a UTF8 argument.
*/
FILE *fossil_fopen(const char *zName, const char *zMode){
#ifdef _WIN32
  wchar_t *uMode = fossil_utf8_to_unicode(zMode);
  wchar_t *uName = fossil_utf8_to_unicode(zName);
  FILE *f = _wfopen(uName, uMode);
  fossil_unicode_free(uName);
  fossil_unicode_free(uMode);
#else
  FILE *f = fopen(zName, zMode);
#endif
  return f;
}







|

|













|

|






1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150

/*
** Return the value of an environment variable as UTF8.
** Use fossil_filename_free() to release resources.
*/
char *fossil_getenv(const char *zName){
#ifdef _WIN32
  wchar_t *uName = fossil_utf8_to_filename(zName);
  void *zValue = _wgetenv(uName);
  fossil_filename_free(uName);
#else
  char *zValue = getenv(zName);
#endif
  if( zValue ) zValue = fossil_filename_to_utf8(zValue);
  return zValue;
}

/*
** Like fopen() but always takes a UTF8 argument.
*/
FILE *fossil_fopen(const char *zName, const char *zMode){
#ifdef _WIN32
  wchar_t *uMode = fossil_utf8_to_unicode(zMode);
  wchar_t *uName = fossil_utf8_to_filename(zName);
  FILE *f = _wfopen(uName, uMode);
  fossil_filename_free(uName);
  fossil_unicode_free(uMode);
#else
  FILE *f = fopen(zName, zMode);
#endif
  return f;
}

Changes to src/rebuild.c.

839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
  DIR *d;
  struct dirent *pEntry;
  Blob aContent; /* content of the just read artifact */
  static int nFileRead = 0;
  void *zUnicodePath;
  char *zUtf8Name;

  zUnicodePath = fossil_utf8_to_unicode(zPath);
  d = opendir(zUnicodePath);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      Blob path;
      char *zSubpath;

      if( pEntry->d_name[0]=='.' ){







|







839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
  DIR *d;
  struct dirent *pEntry;
  Blob aContent; /* content of the just read artifact */
  static int nFileRead = 0;
  void *zUnicodePath;
  char *zUtf8Name;

  zUnicodePath = fossil_utf8_to_filename(zPath);
  d = opendir(zUnicodePath);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      Blob path;
      char *zSubpath;

      if( pEntry->d_name[0]=='.' ){
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
      fflush(stdout);
    }
    closedir(d);
  }else {
    fossil_panic("encountered error %d while trying to open \"%s\".",
                  errno, g.argv[3]);
  }
  fossil_unicode_free(zUnicodePath);
}

/*
** COMMAND: reconstruct*
**
** Usage: %fossil reconstruct FILENAME DIRECTORY
**







|







873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
      fflush(stdout);
    }
    closedir(d);
  }else {
    fossil_panic("encountered error %d while trying to open \"%s\".",
                  errno, g.argv[3]);
  }
  fossil_filename_free(zUnicodePath);
}

/*
** COMMAND: reconstruct*
**
** Usage: %fossil reconstruct FILENAME DIRECTORY
**

Changes to src/utf8.c.

22
23
24
25
26
27
28

29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

105
106
107
108
109
110
111
#include "config.h"
#include "utf8.h"
#include <sqlite3.h>
#ifdef _WIN32
# include <windows.h>
#endif


/*
** Translate MBCS to UTF8.  Return a pointer to the translated text.
** Call fossil_mbcs_free() to deallocate any memory used to store the
** returned pointer when done.
*/
char *fossil_mbcs_to_utf8(const char *zMbcs){
#ifdef _WIN32
  extern char *sqlite3_win32_mbcs_to_utf8(const char*);
  return sqlite3_win32_mbcs_to_utf8(zMbcs);
#else
  return (char*)zMbcs;  /* No-op on unix */
#endif
}

/*
** After translating from UTF8 to MBCS, invoke this routine to deallocate
** any memory used to hold the translation
*/
void fossil_mbcs_free(char *zOld){
#ifdef _WIN32
  sqlite3_free(zOld);
#else
  /* No-op on unix */
#endif
}

/*
** Translate Unicode text into UTF8.
** Return a pointer to the translated text.
** Call fossil_unicode_free() to deallocate any memory used to store the
** returned pointer when done.
*/
char *fossil_unicode_to_utf8(const void *zUnicode){
#ifdef _WIN32
  int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0);
  char *zUtf = sqlite3_malloc( nByte );
  if( zUtf==0 ){
    return 0;
  }
  WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0);
  return zUtf;
#else
  return (char *)zUnicode;  /* No-op on unix */
#endif
}

/*
** Translate UTF8 to unicode for use in system calls.  Return a pointer to the
** translated text..  Call fossil_unicode_free() to deallocate any memory
** used to store the returned pointer when done.
*/
void *fossil_utf8_to_unicode(const char *zUtf8){
#ifdef _WIN32
  int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
  wchar_t *zUnicode = sqlite3_malloc( nByte * 2 );
  if( zUnicode==0 ){
    return 0;
  }
  MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
  return zUnicode;
#else
  return (void *)zUtf8;  /* No-op on unix */
#endif
}

/*
** Deallocate any memory that was previously allocated by
** fossil_unicode_to_utf8().
*/
void fossil_unicode_free(void *pOld){
#ifdef _WIN32
  sqlite3_free(pOld);
#else
  /* No-op on unix */
#endif
}


#if defined(__APPLE__) && !defined(WITHOUT_ICONV)
# include <iconv.h>
#endif

/*
** Translate text from the filename character set into







>






<


<
<
<







<

<
<
<


















|


















|











|


>







22
23
24
25
26
27
28
29
30
31
32
33
34
35

36
37



38
39
40
41
42
43
44

45



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#include "config.h"
#include "utf8.h"
#include <sqlite3.h>
#ifdef _WIN32
# include <windows.h>
#endif

#ifdef _WIN32
/*
** Translate MBCS to UTF8.  Return a pointer to the translated text.
** Call fossil_mbcs_free() to deallocate any memory used to store the
** returned pointer when done.
*/
char *fossil_mbcs_to_utf8(const char *zMbcs){

  extern char *sqlite3_win32_mbcs_to_utf8(const char*);
  return sqlite3_win32_mbcs_to_utf8(zMbcs);



}

/*
** After translating from UTF8 to MBCS, invoke this routine to deallocate
** any memory used to hold the translation
*/
void fossil_mbcs_free(char *zOld){

  sqlite3_free(zOld);



}

/*
** Translate Unicode text into UTF8.
** Return a pointer to the translated text.
** Call fossil_unicode_free() to deallocate any memory used to store the
** returned pointer when done.
*/
char *fossil_unicode_to_utf8(const void *zUnicode){
#ifdef _WIN32
  int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0);
  char *zUtf = sqlite3_malloc( nByte );
  if( zUtf==0 ){
    return 0;
  }
  WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0);
  return zUtf;
#else
  return (char *)zUnicode;  /* TODO: implement for unix */
#endif
}

/*
** Translate UTF8 to unicode for use in system calls.  Return a pointer to the
** translated text..  Call fossil_unicode_free() to deallocate any memory
** used to store the returned pointer when done.
*/
void *fossil_utf8_to_unicode(const char *zUtf8){
#ifdef _WIN32
  int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
  wchar_t *zUnicode = sqlite3_malloc( nByte * 2 );
  if( zUnicode==0 ){
    return 0;
  }
  MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
  return zUnicode;
#else
  return (void *)zUtf8;  /* TODO: implement for unix */
#endif
}

/*
** Deallocate any memory that was previously allocated by
** fossil_unicode_to_utf8().
*/
void fossil_unicode_free(void *pOld){
#ifdef _WIN32
  sqlite3_free(pOld);
#else
  /* TODO: implement for unix */
#endif
}
#endif /* _WIN32 */

#if defined(__APPLE__) && !defined(WITHOUT_ICONV)
# include <iconv.h>
#endif

/*
** Translate text from the filename character set into
147
148
149
150
151
152
153
154
155






















156
157
158
159
160
161
162
163
164
165
166
    zOut = fossil_strdup(zFilename);
  }
  return zOut;
#else
  return (char *)zFilename;  /* No-op on non-mac unix */
#endif
}

/*






















** Deallocate any memory that was previously allocated by
** fossil_filename_to_utf8().
*/
void fossil_filename_free(char *pOld){
#if defined(_WIN32)
  sqlite3_free(pOld);
#elif defined(__APPLE__) && !defined(WITHOUT_ICONV)
  fossil_free(pOld);
#else
  /* No-op on all other unix */
#endif









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|

|







141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
    zOut = fossil_strdup(zFilename);
  }
  return zOut;
#else
  return (char *)zFilename;  /* No-op on non-mac unix */
#endif
}

/*
** Translate UTF8 to unicode for use in filename translations.
** Return a pointer to the translated text..  Call fossil_filename_free()
** to deallocate any memory used to store the returned pointer when done.
**
*/
void *fossil_utf8_to_filename(const char *zUtf8){
#ifdef _WIN32
  int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
  wchar_t *zUnicode = sqlite3_malloc( nByte * 2 );
  if( zUnicode==0 ){
    return 0;
  }
  MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
  return zUnicode;
#elif defined(__APPLE__) && !defined(WITHOUT_ICONV)
  return fossil_strdup(zUtf8);
#else
  return (void *)zUtf8;  /* No-op on unix */
#endif
}

/*
** Deallocate any memory that was previously allocated by
** fossil_filename_to_utf8() or fossil_utf8_to_filename().
*/
void fossil_filename_free(void *pOld){
#if defined(_WIN32)
  sqlite3_free(pOld);
#elif defined(__APPLE__) && !defined(WITHOUT_ICONV)
  fossil_free(pOld);
#else
  /* No-op on all other unix */
#endif

Changes to src/vfile.c.

457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
       "INSERT OR IGNORE INTO sfile(x) SELECT :file"
       "  WHERE NOT EXISTS(SELECT 1 FROM vfile WHERE pathname=:file)"
    );
  }
  depth++;

  zDir = blob_str(pPath);
  zNative = fossil_utf8_to_unicode(zDir);
  d = opendir(zNative);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      char *zPath;
      char *zUtf8;
      if( pEntry->d_name[0]=='.' ){
        if( (scanFlags & SCAN_ALL)==0 ) continue;







|







457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
       "INSERT OR IGNORE INTO sfile(x) SELECT :file"
       "  WHERE NOT EXISTS(SELECT 1 FROM vfile WHERE pathname=:file)"
    );
  }
  depth++;

  zDir = blob_str(pPath);
  zNative = fossil_utf8_to_filename(zDir);
  d = opendir(zNative);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      char *zPath;
      char *zUtf8;
      if( pEntry->d_name[0]=='.' ){
        if( (scanFlags & SCAN_ALL)==0 ) continue;
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
        }
      }
      fossil_filename_free(zUtf8);
      blob_resize(pPath, origSize);
    }
    closedir(d);
  }
  fossil_unicode_free(zNative);

  depth--;
  if( depth==0 ){
    db_finalize(&ins);
  }
}








|







489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
        }
      }
      fossil_filename_free(zUtf8);
      blob_resize(pPath, origSize);
    }
    closedir(d);
  }
  fossil_filename_free(zNative);

  depth--;
  if( depth==0 ){
    db_finalize(&ins);
  }
}