Index: auto.def ================================================================== --- auto.def +++ auto.def @@ -318,11 +318,10 @@ config.make Makefile doc/Doxyfile src/Makefile f-apps/Makefile - th1ish/Makefile cpp/Makefile } foreach {f} $makefiles { makeFromDotIn $f } Index: autosetup/README.autosetup ================================================================== --- autosetup/README.autosetup +++ autosetup/README.autosetup @@ -1,6 +1,6 @@ -README.autosetup created by autosetup v0.7.0+ +README.autosetup created by autosetup v0.6.9 This is the autosetup directory for a local install of autosetup. It contains autosetup, support files and loadable modules. *.tcl files in this directory are optional modules which Index: autosetup/autosetup ================================================================== --- autosetup/autosetup +++ autosetup/autosetup @@ -4,11 +4,11 @@ # vim:se syntax=tcl: # \ dir=`dirname "$0"`; exec "`$dir/autosetup-find-tclsh`" "$0" "$@" # Note that the version has a trailing + on unreleased versions -set autosetup(version) 0.7.0+ +set autosetup(version) 0.6.9 # Can be set to 1 to debug early-init problems set autosetup(debug) [expr {"--debug" in $argv}] ################################################################## @@ -91,17 +91,17 @@ # We simply parse anything that looks like an option set autosetup(getopt) [getopt argv] #"=Core Options:" options-add { - help:=all => "display help and options. Optional: module name, such as --help=system" + help:=local => "display help and options. Optionally specify a module name, such as --help=system" licence license => "display the autosetup license" - version => "display the version of autosetup" + version => "display the version of autosetup" ref:=text manual:=text reference:=text => "display the autosetup command reference. 'text', 'wiki', 'asciidoc' or 'markdown'" - debug => "display debugging output as autosetup runs" - install:=. => "install autosetup to the current or given directory" + debug => "display debugging output as autosetup runs" + install:=. => "install autosetup to the current or given directory" } if {$autosetup(installed)} { # hidden options so we can produce a nice error options-add { sysinstall:path @@ -202,30 +202,24 @@ } } autosetup_add_dep $autosetup(autodef) - # Add $argv to CONFIGURE_OPTS, but ignore duplicates and quote if needed - set configure_opts {} + define CONFIGURE_OPTS "" foreach arg $autosetup(argv) { - set quoted [quote-if-needed $arg] - # O(n^2), but n will be small - if {$quoted ni $configure_opts} { - lappend configure_opts $quoted - } + define-append CONFIGURE_OPTS [quote-if-needed $arg] } - define CONFIGURE_OPTS [join $configure_opts] define AUTOREMAKE [file-normalize $autosetup(exe)] define-append AUTOREMAKE [get-define CONFIGURE_OPTS] # Log how we were invoked configlog "Invoked as: [getenv WRAPPER $::argv0] [quote-argv $autosetup(argv)]" configlog "Tclsh: [info nameofexecutable]" - # Load auto.def as module "auto.def" - autosetup_load_module auto.def source $autosetup(autodef) + # Note that auto.def is *not* loaded in the global scope + source $autosetup(autodef) # Could warn here if options {} was not specified show-notices @@ -346,12 +340,12 @@ } if {![info exists result]} { # No user-specified value. Has options-defaults been set? foreach opt $names { - if {[dict exists $::autosetup(optdefault) $opt]} { - set result [dict get $autosetup(optdefault) $opt] + if {[dict exists $::autosetup(options-defaults) $opt]} { + set result [dict get $autosetup(options-defaults) $opt] } } } if {[info exists result]} { @@ -379,11 +373,11 @@ } # Parse the option definition in $opts and update # ::autosetup(setoptions) and ::autosetup(optionhelp) appropriately # -proc options-add {opts} { +proc options-add {opts {header ""}} { global autosetup # First weed out comment lines set realopts {} foreach line [split $opts \n] { @@ -395,11 +389,12 @@ for {set i 0} {$i < [llength $opts]} {incr i} { set opt [lindex $opts $i] if {[string match =* $opt]} { # This is a special heading - lappend autosetup(optionhelp) [list $opt $autosetup(module)] + lappend autosetup(optionhelp) $opt "" + set header {} continue } unset -nocomplain defaultvalue equal value #puts "i=$i, opt=$opt" @@ -456,12 +451,12 @@ } } else { # String option. lappend autosetup(options) $name - if {$equal ne "="} { - # Was the option given as "name:value=default"? + if {$colon eq ":"} { + # Was ":name=default" given? # If so, set $value to the display name and $defaultvalue to the default # (This is the preferred way to set a default value for a string option) if {[regexp {^([^=]+)=(.*)$} $value -> value defaultvalue]} { dict set autosetup(optdefault) $name $defaultvalue } @@ -471,13 +466,13 @@ if {[dict exists $autosetup(options-defaults) $name]} { # A default was specified with options-defaults, so use it set defaultvalue [dict get $autosetup(options-defaults) $name] dict set autosetup(optdefault) $name $defaultvalue } elseif {![info exists defaultvalue]} { - # No default value was given by value=default or options-defaults - # so use the value as the default when the plain option with no - # value is given (.e.g. just --opt instead of --opt=value) + # For backward compatibility, if ":name" was given, use name as both + # the display text and the default value, but only if the user + # specified the option without the value set defaultvalue $value } if {$equal eq "="} { # String option with optional value @@ -511,23 +506,40 @@ # Now create the help for this option if appropriate if {[lindex $opts $i+1] eq "=>"} { set desc [lindex $opts $i+2] if {[info exists defaultvalue]} { set desc [string map [list @default@ $defaultvalue] $desc] + } + #string match \n* $desc + if {$header ne ""} { + lappend autosetup(optionhelp) $header "" + set header "" } # A multi-line description - lappend autosetup(optionhelp) [list $opthelp $autosetup(module) $desc] + lappend autosetup(optionhelp) $opthelp $desc incr i 2 } } } # @module-options optionlist # -# Deprecated. Simply use 'options' from within a module. +# Like 'options', but used within a module. proc module-options {opts} { - options $opts + set header "" + if {$::autosetup(showhelp) > 1 && [llength $opts]} { + set header "Module Options:" + } + options-add $opts $header + + if {$::autosetup(showhelp)} { + # Ensure that the module isn't executed on --help + # We are running under eval or source, so use break + # to prevent further execution + #return -code break -level 2 + return -code break + } } proc max {a b} { expr {$a > $b ? $a : $b} } @@ -552,21 +564,14 @@ if {$len} { puts "" } } -# Display options (from $autosetup(optionhelp)) for modules that match -# glob pattern $what -proc options-show {what} { - set local 0 +proc options-show {} { # Determine the max option width set max 0 - foreach help $::autosetup(optionhelp) { - lassign $help opt module desc - if {![string match $what $module]} { - continue - } + foreach {opt desc} $::autosetup(optionhelp) { if {[string match =* $opt] || [string match \n* $desc]} { continue } set max [max $max [string length $opt]] } @@ -575,27 +580,17 @@ catch { lassign [exec stty size] rows cols } incr cols -1 # Now output - foreach help $::autosetup(optionhelp) { - lassign $help opt module desc - if {![string match $what $module]} { - continue - } - if {$local == 0 && $module eq "auto.def"} { - puts "Local Options:" - incr local - } + foreach {opt desc} $::autosetup(optionhelp) { if {[string match =* $opt]} { - # Output a special heading line" puts [string range $opt 1 end] continue } puts -nonewline " [format %-${max}s $opt]" if {[string match \n* $desc]} { - # Output a pre-formatted help description as-is puts $desc } else { options-wrap-desc [string trim $desc] $cols " " $indent [expr $max + 2] } } @@ -613,20 +608,16 @@ # # The default is 'name=0', meaning that the option is disabled by default. # If 'name=1' is used to make the option enabled by default, the description should reflect # that with text like "Disable support for ...". # -# An argument option (one which takes a parameter) is of one of the following forms: +# An argument option (one which takes a parameter) is of the form: # -## name:value => "Description of this option" -## name:value=default => "Description of this option with a default value" -## name:=value => "Description of this option with an optional value" +## name:[=]value => "Description of this option" # # If the 'name:value' form is used, the value must be provided with the option (as '--name=myvalue'). -# If the 'name:value=default' form is used, the option has the given default value even if not -# specified by the user. -# If the 'name:=value' form is used, the value is optional and the given value is used +# If the 'name:=value' form is used, the value is optional and the given value is used as the default # if it is not provided. # # The description may contain '@default@', in which case it will be replaced with the default # value for the option (taking into account defaults specified with 'options-defaults'. # @@ -636,26 +627,23 @@ # For example, '--disable-lfs' is an alias for '--disable=largefile': # ## lfs=1 largefile=1 => "Disable large file support" # proc options {optlist} { - global autosetup - - options-add $optlist - - if {$autosetup(showhelp)} { - # If --help, stop now to show help - return -code break - } - - if {$autosetup(module) eq "auto.def"} { - # Check for invalid options - if {[opt-bool option-checking]} { - foreach o [dict keys $::autosetup(getopt)] { - if {$o ni $::autosetup(options)} { - user-error "Unknown option --$o" - } + # Allow options as a list or args + options-add $optlist "Local Options:" + + if {$::autosetup(showhelp)} { + options-show + exit 0 + } + + # Check for invalid options + if {[opt-bool option-checking]} { + foreach o [dict keys $::autosetup(getopt)] { + if {$o ni $::autosetup(options)} { + user-error "Unknown option --$o" } } } } @@ -1183,13 +1171,12 @@ foreach m $args { if {[info exists libmodule($m)]} { continue } set libmodule($m) 1 - if {[info exists modsource(${m}.tcl)]} { - autosetup_load_module $m eval $modsource(${m}.tcl) + automf_load eval $modsource(${m}.tcl) } else { set locs [list ${m}.tcl ${m}/init.tcl] set found 0 foreach dir $dirs { foreach loc $locs { @@ -1205,11 +1192,11 @@ } if {$found} { # For the convenience of the "use" source, point to the directory # it is being loaded from set ::usedir [file dirname $source] - autosetup_load_module $m source $source + automf_load source $source autosetup_add_dep $source } else { autosetup-error "use: No such module: $m" } } @@ -1218,28 +1205,23 @@ proc autosetup_load_auto_modules {} { global autosetup modsource # First load any embedded auto modules foreach mod [array names modsource *.auto] { - autosetup_load_module $mod eval $modsource($mod) + automf_load eval $modsource($mod) } # Now any external auto modules foreach file [glob -nocomplain $autosetup(libdir)/*.auto $autosetup(libdir)/*/*.auto] { - autosetup_load_module [file tail $file] source $file + automf_load source $file } } # Load module source in the global scope by executing the given command -proc autosetup_load_module {module args} { - global autosetup - set prev $autosetup(module) - set autosetup(module) $module - +proc automf_load {args} { if {[catch [list uplevel #0 $args] msg opts] ni {0 2 3}} { autosetup-full-error [error-dump $msg $opts $::autosetup(debug)] } - set autosetup(module) $prev } # Initial settings set autosetup(exe) $::argv0 set autosetup(istcl) 1 @@ -1247,11 +1229,10 @@ set autosetup(installed) 0 set autosetup(sysinstall) 0 set autosetup(msg-checking) 0 set autosetup(msg-quiet) 0 set autosetup(inittypes) {} -set autosetup(module) autosetup # Embedded modules are inserted below here set autosetup(installed) 1 set autosetup(sysinstall) 0 # ----- @module asciidoc-formatting.tcl ----- @@ -1453,26 +1434,26 @@ puts "Usage: [file tail $::autosetup(exe)] \[options\] \[settings\]\n" puts "This is [autosetup_version], a build environment \"autoconfigurator\"" puts "See the documentation online at http://msteveb.github.com/autosetup/\n" - if {$what in {all local}} { - # Need to load auto.def now - if {[file exists $::autosetup(autodef)]} { - # Load auto.def as module "auto.def" - autosetup_load_module auto.def source $::autosetup(autodef) - } - if {$what eq "all"} { - set what * - } else { - set what auto.def - } - } else { - use $what - puts "Options for module $what:" - } - options-show $what + if {$what eq "local"} { + if {[file exists $::autosetup(autodef)]} { + # This relies on auto.def having a call to 'options' + # which will display options and quit + source $::autosetup(autodef) + } else { + options-show + } + } else { + incr ::autosetup(showhelp) + if {[catch {use $what}]} { + user-error "Unknown module: $what" + } else { + options-show + } + } exit 0 } proc autosetup_show_license {} { global modsource autosetup Index: autosetup/autosetup-find-tclsh ================================================================== --- autosetup/autosetup-find-tclsh +++ autosetup/autosetup-find-tclsh @@ -1,15 +1,17 @@ #!/bin/sh # Looks for a suitable tclsh or jimsh in the PATH -# If not found, builds a bootstrap jimsh in current dir from source -# Prefer $autosetup_tclsh if is set in the environment (unless ./jimsh0 works) -d="`dirname "$0"`" -for tclsh in ./jimsh0 $autosetup_tclsh jimsh tclsh tclsh8.5 tclsh8.6 tclsh8.7; do +# If not found, builds a bootstrap jimsh from source +# Prefer $autosetup_tclsh if is set in the environment +d=`dirname "$0"` +{ "$d/jimsh0" "$d/autosetup-test-tclsh"; } 2>/dev/null && exit 0 +PATH="$PATH:$d"; export PATH +for tclsh in $autosetup_tclsh jimsh tclsh tclsh8.5 tclsh8.6; do { $tclsh "$d/autosetup-test-tclsh"; } 2>/dev/null && exit 0 done echo 1>&2 "No installed jimsh or tclsh, building local bootstrap jimsh0" for cc in ${CC_FOR_BUILD:-cc} gcc; do - { $cc -o jimsh0 "$d/jimsh0.c"; } 2>/dev/null || continue - ./jimsh0 "$d/autosetup-test-tclsh" && exit 0 + { $cc -o "$d/jimsh0" "$d/jimsh0.c"; } 2>/dev/null || continue + "$d/jimsh0" "$d/autosetup-test-tclsh" && exit 0 done echo 1>&2 "No working C compiler found. Tried ${CC_FOR_BUILD:-cc} and gcc." echo false Index: autosetup/local.tcl ================================================================== --- autosetup/local.tcl +++ autosetup/local.tcl @@ -1,7 +1,8 @@ -# For this project, disable the pager for --help -set useropts(nopager) 1 +# For this project, disable the pager for --help and --ref +# The user can still enable by using --nopager=0 or --disable-nopager +dict set autosetup(optdefault) nopager 1 # Searches for a usable Tcl (prefer 8.6, 8.5, 8.4) in the given paths # Returns a dictionary of the contents of the tclConfig.sh file, or # empty if not found proc parse-tclconfig-sh {args} { Index: f-apps/f-mfparse.c ================================================================== --- f-apps/f-mfparse.c +++ f-apps/f-mfparse.c @@ -131,11 +131,11 @@ f_out("Dumping mf to file [%s]\n", ofile); rc = fsl_buffer_to_filename(&bout, ofile); assert(!rc); { fsl_buffer sha = fsl_buffer_empty; - rc = fsl_sha1sum_filename(ofile, &sha); + rc = fsl_sha3sum_filename(ofile, &sha); assert(!rc); f_out("SHA of [%s] = [%b]\n", ofile, &sha); seemsSafeEnough = (0==fsl_strcmp(fsl_buffer_cstr(&sha), mf.uuid)); f_out("SHA match? %s\n", Index: f-apps/f-timeline.c ================================================================== --- f-apps/f-timeline.c +++ f-apps/f-timeline.c @@ -101,11 +101,11 @@ "ORDER BY filename.name %s", fsl_cx_filename_collation(f)); if(rc){ return fsl_cx_uplift_db_error(f, db); } - rc = fsl_stmt_bind_text(st, 1, uuid, FSL_UUID_STRLEN, 0); + rc = fsl_stmt_bind_text(st, 1, uuid, -1, 0); assert(0==rc); while(FSL_RC_STEP_ROW==(rc=fsl_stmt_step(st))){ char const * changeType; if(!doneHead){ doneHead = 1; Index: f-apps/test.c ================================================================== --- f-apps/test.c +++ f-apps/test.c @@ -114,14 +114,13 @@ rc = fsl_sym_to_uuid(f, "current", FSL_CATYPE_CHECKIN, &uuid, &id2); assert(0==rc); assert(uuid); assert(id2 == id); - assert(FSL_UUID_STRLEN==fsl_strlen(uuid)); - f_out("Current checkout: %.*s\n", FSL_UUID_STRLEN, uuid); - fsl_free(uuid); - + assert(fsl_is_uuid(uuid)); + f_out("Current checkout: %s\n", uuid); + fsl_free(uuid); } return rc; } /* static */ int test_leaves_rebuild(){ @@ -150,11 +149,11 @@ fsl_cx_err_report(f, 1); assert(!rc); assert(1032==c.used); rc = fsl_sha1sum_buffer( &c, &c ); assert(!rc); - assert(FSL_UUID_STRLEN==c.used); + assert(fsl_is_uuid_len((int)c.used)); assert(0==fsl_strcmp( sym, fsl_buffer_cstr(&c) )); /* Now fetch a few other versions of that same file and ensure that they meet our expectations... */ @@ -167,11 +166,11 @@ rc = fsl_content_get(f, rid, &c); assert(!rc); assert(2076==c.used); rc = fsl_sha1sum_buffer( &c, &c ); assert(!rc); - assert(FSL_UUID_STRLEN==c.used); + assert(fsl_is_uuid_len((int)c.used)); assert(0==fsl_strcmp( sym, fsl_buffer_cstr(&c) )); sym = "f7d3a2e155d59a96ecc37001b05de26dee23c0cf"; rid = 0; rc = fsl_sym_to_rid(f, sym, FSL_CATYPE_ANY, &rid); @@ -180,11 +179,11 @@ rc = fsl_content_get(f, rid, &c); assert(!rc); assert(2481==c.used); rc = fsl_sha1sum_buffer( &c, &c ); assert(!rc); - assert(FSL_UUID_STRLEN==c.used); + assert(fsl_is_uuid_len((int)c.used)); assert(0==fsl_strcmp( sym, fsl_buffer_cstr(&c) )); sym = "31e01e2a3c"; rid = 0; Index: include/fossil-scm/fossil-content.h ================================================================== --- include/fossil-scm/fossil-content.h +++ include/fossil-scm/fossil-content.h @@ -2165,11 +2165,11 @@ /** Return true if the given artifact ID should is listed in f's shun table, else false. */ -FSL_EXPORT char fsl_uuid_is_shunned(fsl_cx * f, fsl_uuid_cstr zUuid); +FSL_EXPORT int fsl_uuid_is_shunned(fsl_cx * f, fsl_uuid_cstr zUuid); /** Given a fsl_cx with an opened checkout, and a filename, this function canonicalizes zOrigName to a form suitable for use as an in-repo filename, _appending_ the results to pOut. If pOut is @@ -2996,12 +2996,33 @@ except that (A) bIn is const in this call and non-const in the other form (due to cursor traversal requirements) and (B) it returns FSL_RC_MISUSE if pIn is NULL. */ FSL_EXPORT int fsl_repo_import_buffer( fsl_cx * f, fsl_buffer const * bIn, - fsl_id_t * rid, fsl_uuid_str * uuid ); + fsl_id_t * rid, fsl_uuid_str * uuid ); + + +/** + Hashes all of pIn, appending the hash to pOut. Returns 0 on succes, + FSL_RC_OOM if allocation of space in pOut fails. The hash algorithm + used depends on the given fossil context's current hash policy and + the value of the 2nd argument: + + If the 2nd argument is false, the hash is performed per the first + argument's policy. If the 2nd argument is true, the hash policy is + effectively inverted. e.g. if the context prefers SHA3 hashes, the + alternate form will use SHA1. + + Returns FSL_RC_RANGE if the hash is not possible due to conflicting + values for the policy and its alternate (e.g. a context with policy + FSL_HPOLICY_SHA3_ONLY will refuse to apply an SHA1 hash). + Returns 0 on success. +*/ +FSL_EXPORT int fsl_cx_hash_buffer( const fsl_cx * f, int useAlternate, + fsl_buffer const * pIn, + fsl_buffer * pOut); #if 0 /** NOT YET IMPLEMENTED - just thinking out loud here. */ Index: include/fossil-scm/fossil-hash.h ================================================================== --- include/fossil-scm/fossil-hash.h +++ include/fossil-scm/fossil-hash.h @@ -33,20 +33,52 @@ /** Various set-in-stone constants used by the API. */ enum fsl_hash_constants { /** - The length, in bytes, of fossil's hex-form UUID strings. - - FIXME: SHA3 + The length, in bytes, of fossil's hex-form SHA1 UUID strings. +*/ +FSL_UUIDv1_STRLEN = 40, +/** + The length, in bytes, of fossil's hex-form SHA3-256 UUID strings. */ -FSL_UUID_STRLEN = 40, +FSL_UUIDv2_STRLEN = 64, /** The length, in bytes, of a hex-form MD5 hash. */ -FSL_MD5_STRLEN = 32 +FSL_MD5_STRLEN = 32, + +FSL_UUID_STRLEN_MIN = FSL_UUIDv1_STRLEN, +FSL_UUID_STRLEN_MAX = FSL_UUIDv2_STRLEN +}; + +/** + Internal IDs for artifact hash types the library works. + */ +enum fsl_hash_types_t { +/** Invalid hash type. */ +FSL_HTYPE_ERROR = 0, +/** SHA1. */ +FSL_HTYPE_SHA1 = 1, +/** SHA3-256. */ +FSL_HTYPE_K256 = 2 +}; +typedef enum fsl_hash_types_t fsl_hash_types_t; + +enum fsl_hash_policy_t { +/* Use SHA1 hashes */ +FSL_HPOLICY_SHA1 = 0, +/* SHA1 but auto-promote to SHA3 */ +FSL_HPOLICY_AUTO = 1, +/* Use SHA3 hashes */ +FSL_HPOLICY_SHA3 = 2, +/* Use SHA3 hashes exclusively */ +FSL_HPOLICY_SHA3_ONLY = 3, +/* Shun all SHA1 objects */ +FSL_HPOLICY_SHUN_SHA1 = 4 }; +typedef enum fsl_hash_policy_t fsl_hash_policy_t; typedef struct fsl_md5_cx fsl_md5_cx; typedef struct fsl_sha1_cx fsl_sha1_cx; typedef struct fsl_sha3_cx fsl_sha3_cx; @@ -213,11 +245,11 @@ Holds state for SHA1 calculations. It is intended to be used like this: @code unsigned char digest[20] - char hex[FSL_UUID_STRLEN+1]; + char hex[FSL_UUIDv1_STRLEN+1]; fsl_sha1_cx cx = fsl_sha1_cx_empty; // alternately: fsl_sha1_init(&cx) ...call fsl_sha1_update(&cx,...) any number of times to ...incrementally calculate the hash. fsl_sha1_final(&cx, digest); // ends the calculation @@ -302,15 +334,23 @@ @see fsl_sha1_update() @see fsl_sha1_digest_to_base16() */ FSL_EXPORT int fsl_sha1_final(fsl_sha1_cx *context, unsigned char * digest); +/** + A convenience form of fsl_sha1_final() which writes + FSL_UUIDv1_STRLEN+1 bytes (hash plus terminating NUL byte) to the + 2nd argument and returns a (const char *)-type cast of the 2nd + argument. +*/ +FSL_EXPORT const char * fsl_sha1_final_hex(fsl_sha1_cx *context, char * zHex); + /** Convert a digest into base-16. digest must be at least 20 bytes - long and hold an SHA1 digest. zBuf must be at least (FSL_UUID_STRLEN - + 1) bytes long, for FSL_UUID_STRLEN characters of - hexidecimal-form SHA1 hash and 1 NUL byte. + long and hold an SHA1 digest. zBuf must be at least (FSL_UUIDv1_STRLEN + + 1) bytes long, to which FSL_UUIDv1_STRLEN characters of + hexidecimal-form SHA1 hash and 1 NUL byte will be written. @see fsl_sha1_final() */ FSL_EXPORT void fsl_sha1_digest_to_base16(unsigned char *digest, char *zBuf); @@ -373,11 +413,13 @@ FSL_SHA3_INVALID = 0, FSL_SHA3_128 = 128, FSL_SHA3_160 = 160, FSL_SHA3_192 = 192, FSL_SHA3_224 = 224, FSL_SHA3_256 = 256, FSL_SHA3_288 = 288, FSL_SHA3_320 = 320, FSL_SHA3_352 = 352, FSL_SHA3_384 = 384, FSL_SHA3_416 = 416, FSL_SHA3_448 = 448, FSL_SHA3_480 = 480, -FSL_SHA3_512 = 512 +FSL_SHA3_512 = 512, +/* Default SHA3 flavor */ +FSL_SHA3_DEFAULT = 256 }; /** Type for holding SHA3 processing state. Each instance must be initialized with fsl_sha3_init(), populated with fsl_sha3_update(), @@ -385,11 +427,11 @@ Sample usage: @code fsl_sha3_cx cx; - fsl_sha3_init(&cx, FSL_SHA3_256); + fsl_sha3_init(&cx, FSL_SHA3_DEFAULT); fsl_sha3_update(&cx, memory, lengthOfMemory); fsl_sha3_end(&cx); printf("Hash = %s\n", (char const *)cx.hex); @endcode @@ -418,17 +460,23 @@ @see fsl_sha3_init() */ FSL_EXPORT enum fsl_sha3_hash_size fsl_sha3_hash_size_for_int(int); /** - Initialize a new hash. The second argument specifies the size of the hash - in bits. Results are undefined if cx is NULL or sz is not a valid value. + Initialize a new hash. The second argument specifies the size of + the hash in bits. Results are undefined if cx is NULL or sz is not + a valid positive value. After calling this, use fsl_sha3_update() to hash data and fsl_sha3_end() to finalize the hashing process and generate a digest. */ -FSL_EXPORT void fsl_sha3_init(fsl_sha3_cx *cx, enum fsl_sha3_hash_size sz); +FSL_EXPORT void fsl_sha3_init2(fsl_sha3_cx *cx, enum fsl_sha3_hash_size sz); + +/** + Equivalent to fsl_sha3_init2(cx, FSL_SHA3_DEFAULT). +*/ +FSL_EXPORT void fsl_sha3_init(fsl_sha3_cx *cx); /** Updates cx's state to include the first len bytes of data. If cx is NULL results are undefined (segfault!). If mem is not @@ -439,29 +487,89 @@ @see fsl_sha3_end() */ FSL_EXPORT void fsl_sha3_update( fsl_sha3_cx *cx, void const *data, unsigned int len); /** - To be called when hashing is complete: finishes the hash + To be called when SHA3 hashing is complete: finishes the hash calculation and populates cx->hex with the final hash code in hexidecimal-string form. Returns the binary-form digest value, which refers to cx->size/8 bytes of memory which lives in the cx object. After this call cx->hex will be populated with cx->size/4 bytes of lower-case ASCII hex codes plus a terminating NUL byte. + + Potential TODO: change fsl_sha1_final() and fsl_md5_final() to use + these same return semantics. @see fsl_sha3_init() @see fsl_sha3_update() */ FSL_EXPORT unsigned char const * fsl_sha3_end(fsl_sha3_cx *cx); -/* TODOs: port the sha1 counterparts of these to sha3: */ -/* TODO */FSL_EXPORT void fsl_sha3_digest_to_base16(unsigned char *digest, char *zBuf); -/* TODO */FSL_EXPORT int fsl_sha3sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum); -/* TODO */FSL_EXPORT char *fsl_sha3sum_cstr(const char *zIn, fsl_int_t len); -/* TODO */FSL_EXPORT int fsl_sha3sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum); -/* TODO */FSL_EXPORT int fsl_sha1sum_filename(const char *zFilename, fsl_buffer *pCksum); + +/** + SHA3-256 counterpart of fsl_sha1_digest_to_base16(). digest must be at least + 32 bytes long and hold an SHA3 digest. zBuf must be at least (FSL_UUIDv2_STRLEN+1) + bytes long, to which FSL_UUIDv2_STRLEN characters of + hexidecimal-form SHA3 hash and 1 NUL byte will be written + + @see fsl_sha3_end(). +*/ +FSL_EXPORT void fsl_sha3_digest_to_base16(unsigned char *digest, char *zBuf); +/** + SHA3 counter part of fsl_sha1sum_buffer(). +*/ +FSL_EXPORT int fsl_sha3sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum); +/** + SHA3 counter part of fsl_sha1sum_cstr(). +*/ +FSL_EXPORT char *fsl_sha3sum_cstr(const char *zIn, fsl_int_t len); +/** + SHA3 counterpart of fsl_sha1sum_stream(). + */ +FSL_EXPORT int fsl_sha3sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum); +/** + SHA3 counterpart of fsl_sha1sum_filename(). + */ +FSL_EXPORT int fsl_sha3sum_filename(const char *zFilename, fsl_buffer *pCksum); + +/** + Expects zHash to be a full-length hash value of one of the + fsl_hash_types_t-specified types, and nHash to be the length, in + bytes, of zHash's contents (which must be the full hash length, not + a prefix). If zHash can be validated as a hash, its corresponding + hash type is returned, else FSL_HTYPE_ERROR is returned. +*/ +FSL_EXPORT fsl_hash_types_t fsl_validate_hash(const char *zHash, int nHash); + +/** + Expects (zHash, nHash) to refer to a full hash (of a supported + content hash type) of pIn's contents. This routine hashes pIn's + contents and, if it compares equivalent to zHash then the ID of the + hash type is returned. On a mismatch, FSL_HTYPE_ERROR is returned. +*/ +FSL_EXPORT fsl_hash_types_t fsl_verify_blob_hash(fsl_buffer const * pIn, + const char *zHash, int nHash); + +/** + Sets f's hash policy and returns the previous value. +*/ +FSL_EXPORT fsl_hash_policy_t fsl_cx_hash_policy_set(fsl_cx *f, fsl_hash_policy_t p); +/** + Returns f's current hash policy. +*/ +FSL_EXPORT fsl_hash_policy_t fsl_cx_hash_policy_get(fsl_cx const*f); +/** + Returns a human-friendly name for f's current hash policy. +*/ +FSL_EXPORT char const * fsl_cx_hash_policy_name(fsl_cx const*f); + +/** + Returns a human-readable name for the given hash type, or its + second argument h is not a supported hash type. + */ +FSL_EXPORT const char * fsl_hash_type_name(fsl_hash_types_t h, const char *zUnknown); #if defined(__cplusplus) } /*extern "C"*/ #endif #endif /* NET_FOSSIL_SCM_FSL_HASH_H_INCLUDED */ Index: include/fossil-scm/fossil-internal.h ================================================================== --- include/fossil-scm/fossil-internal.h +++ include/fossil-scm/fossil-internal.h @@ -548,10 +548,11 @@ /** The "project-code" config option. */ char * projectCode; + fsl_hash_policy_t hashPolicy; /** Holds various glob lists. */ struct { /** @@ -668,15 +669,16 @@ fsl_id_bag_empty_m/*mfSeen*/, \ fsl_id_bag_empty_m/*leafCheck*/, \ fsl_id_bag_empty_m/*toVerify*/, \ 0/*mtimeManifest*/, \ NULL/*projectCode*/, \ + FSL_HPOLICY_AUTO/*hashPolicy*/, \ {/*globs*/ \ fsl_list_empty_m/*ignore*/, \ fsl_list_empty_m/*binary*/, \ fsl_list_empty_m/*crnl*/ \ - } \ + } \ }, \ {/*ticket*/ \ fsl_list_empty_m/*customFields*/, \ 0/*hasTicket*/, \ 0/*hasCTime*/, \ Index: include/fossil-scm/fossil-util.h ================================================================== --- include/fossil-scm/fossil-util.h +++ include/fossil-scm/fossil-util.h @@ -59,25 +59,25 @@ */ typedef int (*fsl_generic_cmp_f)( void const * lhs, void const * rhs ); /** fsl_uuid_str and fsl_uuid_cstr are "for documentation and - readability purposes" typedefs used to denote strings which the - API requires to be in the form of Fossil UUID strings. Such - strings are exactly FSL_UUID_STRLEN bytes long plus a + readability purposes" typedefs used to denote strings which the API + requires to be in the form of Fossil UUID strings. Such strings are + exactly FSL_UUIDv1_STRLEN or FSL_UUIDv2_STRLEN bytes long plus a terminating NUL byte and contain only lower-case hexadecimal - bytes. Where this typedef is used, the library requires, - enforces, and/or assumes (at different times) that fsl_is_uuid() - returns true for such strings (if they are not NULL, though not - all contexts allow a NULL UUID). These typedef are _not_ used to - denote arguments which may refer to partial UUIDs or symbolic - names, only 100% bonafide Fossil UUIDs (which are different from - RFC4122 UUIDs). + bytes. Where this typedef is used, the library requires, enforces, + and/or assumes (at different times) that fsl_is_uuid() returns true + for such strings (if they are not NULL, though not all contexts + allow a NULL UUID). These typedef are _not_ used to denote + arguments which may refer to partial UUIDs or symbolic names, only + 100% bonafide Fossil UUIDs (which are different from RFC4122 + UUIDs). The API guarantees that this typedef will always be (char *) and that fsl_uuid_cstr will always ben (char const *), and thus it - is safe/portable to use those type instead of thse. These + is safe/portable to use those type instead of these. These typedefs serve only to improve the readability of certain APIs by implying (through the use of this typedef) the preconditions defined for UUID strings. @see fsl_is_uuid() @@ -90,18 +90,18 @@ @see fsl_is_uuid() */ typedef char const * fsl_uuid_cstr; /** - Returns true (non-0) if str is not NULL, is exactly - FSL_UUID_STRLEN bytes long (meaning its final byte is a NUL), - and contains only lower-case hexadecimal characters, else - returns false (0). - - Note that Fossil UUIDs are not RFC4122 UUIDs, but are SHA1 - hash strings. Don't let that disturb you. As Tim Berners-Lee - writes: + If the NUL-terminated input str is exactly FSL_UUIDv1_STRLEN or + FSL_UUIDv2_STRLEN bytes long and contains only lower-case + hexadecimal characters, returns the length of the string, else + returns 0. + + Note that Fossil UUIDs are not RFC4122 UUIDs, but are SHA1 or + SHA3-256 hash strings. Don't let that disturb you. As Tim + Berners-Lee writes: 'The assertion that the space of URIs is a universal space sometimes encounters opposition from those who feel there should not be one universal space. These people need not oppose the concept because it is not of a single universal space: Indeed, @@ -116,11 +116,16 @@ Source: https://www.w3.org/DesignIssues/Axioms.html (Just mentally translate URI as UUID.) */ -FSL_EXPORT char fsl_is_uuid(char const * str); +FSL_EXPORT int fsl_is_uuid(char const * str); + +/** + If x is a valid fossil UUID length, it is returned, else 0 is returned. +*/ +FSL_EXPORT int fsl_is_uuid_len(int x); /** Expects str to be a string containing an unsigned decimal value. Returns its decoded value, or -1 on error. */ @@ -1454,11 +1459,21 @@ given strings, case-sensitively. Returns 0 if nByte is 0. */ FSL_EXPORT int fsl_strncmp(const char *zA, const char *zB, fsl_size_t nByte); /** - Equivalent to fsl_strncmp(lhs, rhs, FSL_UUID_STRLEN). + Equivalent to fsl_strncmp(lhs, rhs, X), where X is either + FSL_UUIDv1_STRLEN or FSL_UUIDv2_STRLEN: if both lhs and rhs are + longer than FSL_UUIDv1_STRLEN then they are assumed to be + FSL_UUIDv2_STRLEN bytes long and are compared as such, else they + are assumed to be FSL_UUIDv1_STRLEN bytes long and compared as + such. + + Potential FIXME/TODO: if their lengths differ, i.e. one is v1 and + one is v2, compare them up to their common length then, if they + still compare equivalent, treat the shorter one as less-than the + longer. */ FSL_EXPORT int fsl_uuidcmp( fsl_uuid_cstr lhs, fsl_uuid_cstr rhs ); /** Returns false if s is NULL or starts with any of (0 (NUL), '0' Index: src/fsl.c ================================================================== --- src/fsl.c +++ src/fsl.c @@ -185,17 +185,29 @@ }else{ return realloc(mem, n); } } -char fsl_is_uuid(char const * str){ +int fsl_is_uuid(char const * str){ fsl_size_t const len = fsl_strlen(str); - return (FSL_UUID_STRLEN==len) - && fsl_validate16(str, FSL_UUID_STRLEN); + if(FSL_UUIDv1_STRLEN==len){ + return fsl_validate16(str, FSL_UUIDv1_STRLEN) ? FSL_UUIDv1_STRLEN : 0; + }else if(FSL_UUIDv2_STRLEN==len){ + return fsl_validate16(str, FSL_UUIDv2_STRLEN) ? FSL_UUIDv2_STRLEN : 0; + }else{ + return 0; + } } - - +int fsl_is_uuid_len(int x){ + switch(x){ + case FSL_UUIDv1_STRLEN: + case FSL_UUIDv2_STRLEN: + return x; + default: + return 0; + } +} void fsl_error_clear( fsl_error * err ){ if(err){ fsl_buffer_clear(&err->msg); *err = fsl_error_empty; } @@ -370,11 +382,17 @@ } } int fsl_uuidcmp( fsl_uuid_cstr lhs, fsl_uuid_cstr rhs ){ - return fsl_strncmp( lhs, rhs, FSL_UUID_STRLEN ); + if(lhs[FSL_UUIDv1_STRLEN] && rhs[FSL_UUIDv1_STRLEN]){ + return fsl_strncmp( lhs, rhs, FSL_UUIDv2_STRLEN); + }else if(!lhs[FSL_UUIDv1_STRLEN] && !rhs[FSL_UUIDv1_STRLEN]){ + return fsl_strncmp( lhs, rhs, FSL_UUIDv1_STRLEN ); + }else{ + return fsl_strcmp(lhs, rhs); + } } int fsl_strnicmp(const char *zA, const char *zB, fsl_int_t nByte){ if( zA==0 ){ if( zB==0 ) return 0; Index: src/fsl_auth.c ================================================================== --- src/fsl_auth.c +++ src/fsl_auth.c @@ -46,11 +46,11 @@ fsl_sha1_update(&hash, zLoginName, fsl_strlen(zLoginName)); fsl_sha1_update(&hash, "/", 1); fsl_sha1_update(&hash, zPw, fsl_strlen(zPw)); fsl_sha1_final(&hash, zResult); fsl_sha1_digest_to_base16(zResult, zDigest); - return fsl_strndup( zDigest, FSL_UUID_STRLEN ); + return fsl_strndup( zDigest, FSL_UUIDv1_STRLEN ); } } FSL_EXPORT char * fsl_repo_login_group_name(fsl_cx * f){ return f Index: src/fsl_content.c ================================================================== --- src/fsl_content.c +++ src/fsl_content.c @@ -16,13 +16,14 @@ https://www.hwaci.com/drh/ ***************************************************************************** This file houses the code for the fsl_content_xxx() APIS. */ -#include #include "fossil-scm/fossil-internal.h" +#include +#include /* memcmp() */ /* Only for debugging */ #include #define MARKER(pfexp) \ do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ @@ -330,26 +331,44 @@ fsl_buffer hash = fsl_buffer_empty; char markAsUnclustered = 0; char markAsUnsent = 1; char isDephantomize = 0; fsl_db * dbR = fsl_cx_db_repo(f); - int rc; + int const zUuidLen = zUuid ? fsl_is_uuid(zUuid) : 0; + int rc = 0; char inTrans = 0; assert(f); assert(dbR); assert(pBlob); assert(srcId==0 || zUuid!=NULL); - assert(!zUuid || fsl_is_uuid(zUuid)); + assert(!zUuid || zUuidLen); if(!dbR) return FSL_RC_NOT_A_REPO; if(!zUuid){ assert(0==uncompSize); - rc = fsl_sha1sum_buffer(pBlob, &hash); + /* "auxiliary hash" bits from: + https://fossil-scm.org/fossil/file?ci=c965636958eb58aa&name=src%2Fcontent.c&ln=527-537 + */ + /* First check the auxiliary hash to see if there is already an artifact + ** that uses the auxiliary hash name */ + rc = fsl_cx_hash_buffer(f, 1, pBlob, &hash); + if(rc) goto end; + assert(hash.used>=FSL_UUIDv1_STRLEN); + rid = fsl_uuid_to_rid(f, fsl_buffer_cstr(&hash)); + assert(rid>=0 && "Cannot have malformed/ambiguous UUID at this point."); + if(!rid){ + /* No existing artifact with the auxiliary hash name. Therefore, use + ** the primary hash name. */ + hash.used = 0; + rc = fsl_cx_hash_buffer(f, 0, pBlob, &hash); + if(rc) goto end; + assert(hash.used>=FSL_UUIDv1_STRLEN); + } }else{ - rc = fsl_buffer_append(&hash, zUuid, FSL_UUID_STRLEN); + rc = fsl_buffer_append(&hash, zUuid, zUuidLen); + if(rc) goto end; } - if(rc) goto end; - + assert(!rc); if(uncompSize){ /* pBlob is assumed to be compressed. */ assert(fsl_buffer_is_compressed(pBlob)); size = uncompSize; }else{ @@ -360,11 +379,15 @@ } } rc = fsl_db_transaction_begin(dbR); if(rc) goto end; inTrans = 1; - + if( f->cache.hashPolicy==FSL_HPOLICY_AUTO && hash.used>FSL_UUIDv1_STRLEN ){ + f->cache.hashPolicy = FSL_HPOLICY_SHA3; + rc = fsl_config_set_int32(f, FSL_CONFDB_REPO, "hash-policy", FSL_HPOLICY_SHA3); + if(rc) goto end; + } /* Check to see if the entry already exists and if it does whether or not the entry is a phantom. */ rc = fsl_db_prepare_cached(dbR, &s1, "SELECT rid, size FROM blob WHERE uuid=?"); @@ -531,11 +554,11 @@ rc = fsl_content_mark_available(f, rid); if(rc) goto end; } if( isDephantomize ){ #if 0 - /* MISSING */ + /* FIXME?: MISSING */ after_dephantomize(rid, 0); #else assert(!"Missing code: after_dephantomize()"); #endif } @@ -559,11 +582,10 @@ } rc = fsl_repo_verify_before_commit(f, rid); if(rc) goto end /* FSL_RC_OOM is basically the "only possible" failure after this point. */; - /* Code after end: relies on the following 2 lines: */ rc = fsl_db_transaction_end(dbR, 0); inTrans = 0; if(!rc){ if(outRid) *outRid = rid; @@ -577,11 +599,10 @@ if(!uncompSize){ fsl_buffer_clear(&cmpr); }/* else cmpr.mem (if any) belongs to pBlob */ return rc; } - char fsl_acache_expire_oldest(fsl_acache * c){ fsl_int_t i; fsl_int_t mnAge = c->nextAge; fsl_int_t mn = -1; @@ -684,38 +705,14 @@ return fsl_cx_err_set(f, FSL_RC_CONSISTENCY, "Serious problem: delta-loop in repository"); } int fsl_content_put( fsl_cx * f, fsl_buffer const * pBlob, fsl_id_t * newRid){ -#if 1 return fsl_content_put_ex(f, pBlob, NULL, 0, 0, 0, newRid); -#else - /* - EXPERIMENT: if pBlob appears to be compressed, pass the proper - uncompressed size value (and required UUID) to put_ex(). - - Aaarrggg - there's the catch. We cannot know the UUID without - decompressing the data. - */ - fsl_int_t const ucSize = fsl_buffer_is_compressed(pBlob) - ? fsl_buffer_uncompressed_size(pBlob) - : 0; - if(ucSize < 0) return FSL_RC_RANGE; - else{ - fsl_buffer uuid = fsl_buffer_empty; - int rc = fsl_sha1sum_buffer(pBlob, &uuid); - if(!rc){ - rc = fsl_content_put_ex(f, pBlob, fsl_buffer_cstr(&uuid), 0, - (fsl_size_t)ucSize, 0, newRid); - } - fsl_buffer_clear(&uuid); - return rc; - } -#endif -} - -char fsl_uuid_is_shunned(fsl_cx * f, fsl_uuid_cstr zUuid){ +} + +int fsl_uuid_is_shunned(fsl_cx * f, fsl_uuid_cstr zUuid){ fsl_int32_t i = 0; fsl_db * db = fsl_cx_db_repo(f); if( !db || zUuid==0 || zUuid[0]==0 ) return 0; i = fsl_db_g_int32( db, 0, "SELECT 1 FROM shun WHERE uuid=%Q", @@ -728,12 +725,13 @@ fsl_id_t * newId ){ fsl_id_t rid = 0; int rc; fsl_db * db = fsl_cx_db_repo(f); fsl_stmt * s1 = NULL, * s2 = NULL; + int const uuidLen = uuid ? fsl_is_uuid(uuid) : 0; if(!f || !uuid) return FSL_RC_MISUSE; - else if(!fsl_is_uuid(uuid)) return FSL_RC_RANGE; + else if(!uuidLen) return FSL_RC_RANGE; if(!db) return FSL_RC_NOT_A_REPO; if( fsl_uuid_is_shunned(f, uuid) ){ return fsl_cx_err_set(f, FSL_RC_ACCESS, "UUID is shunned: %s", uuid) /* need new error code? */; @@ -743,11 +741,11 @@ rc = fsl_db_prepare_cached(db, &s1, "INSERT INTO blob(rcvid,size,uuid,content)" "VALUES(0,-1,?,NULL)"); if(rc) goto end; - rc = fsl_stmt_bind_text(s1, 1, uuid, FSL_UUID_STRLEN, 0); + rc = fsl_stmt_bind_text(s1, 1, uuid, uuidLen, 0); if(!rc) rc = fsl_stmt_step(s1); fsl_stmt_cached_yield(s1); if(FSL_RC_STEP_DONE!=rc) goto end; else rc = 0; rid = fsl_db_last_insert_id(db); @@ -1308,7 +1306,57 @@ fsl_buffer_reset(canon); if(rc && inTrans) fsl_db_transaction_rollback(db); return rc; } +fsl_hash_types_t fsl_validate_hash(const char *zHash, int nHash){ + /* fossil(1) counterpart: hname_validate() */ + fsl_hash_types_t rc; + switch(nHash){ + case FSL_UUIDv1_STRLEN: rc = FSL_HTYPE_SHA1; break; + case FSL_UUIDv2_STRLEN: rc = FSL_HTYPE_K256; break; + default: return FSL_HTYPE_ERROR; + } + return fsl_validate16(zHash, (fsl_size_t)nHash) ? rc : FSL_HTYPE_ERROR; +} + +const char * fsl_hash_type_name(fsl_hash_types_t h, const char *zUnknown){ + /* fossil(1) counterpart: hname_alg() */ + switch(h){ + case FSL_HTYPE_SHA1: return "SHA1"; + case FSL_HTYPE_K256: return "SHA3-256"; + default: return zUnknown; + } +} +fsl_hash_types_t fsl_verify_blob_hash(fsl_buffer const * pIn, + const char *zHash, int nHash){ + fsl_hash_types_t id = FSL_HTYPE_ERROR; + switch(nHash){ + case FSL_UUIDv1_STRLEN:{ + fsl_sha1_cx cx; + char hex[FSL_UUIDv1_STRLEN+1] = {0}; + fsl_sha1_init(&cx); + fsl_sha1_update(&cx, pIn->mem, (unsigned)pIn->used); + fsl_sha1_final_hex(&cx, hex); + if(0==memcmp(hex, zHash, FSL_UUIDv1_STRLEN)){ + id = FSL_HTYPE_SHA1; + } + break; + } + case FSL_UUIDv2_STRLEN:{ + fsl_sha3_cx cx; + unsigned char const * hex; + fsl_sha3_init(&cx); + fsl_sha3_update(&cx, pIn->mem, (unsigned)pIn->used); + hex = fsl_sha3_end(&cx); + if(0==memcmp(hex, zHash, FSL_UUIDv2_STRLEN)){ + id = FSL_HTYPE_K256; + } + break; + } + default: + break; + } + return id; +} #undef MARKER Index: src/fsl_cx.c ================================================================== --- src/fsl_cx.c +++ src/fsl_cx.c @@ -954,15 +954,22 @@ NULL); if(!rc && !(FSL_CX_F_IS_OPENING_CKOUT & f->flags)){ rc = fsl_cx_after_open(f); } if(!rc){ + fsl_db * const db = fsl_cx_db_repo(f); fsl_cx_username_from_repo(f); f->cache.allowSymlinks = fsl_config_get_bool(f, FSL_CONFDB_REPO, f->cache.allowSymlinks, "allow-symlinks"); + if(fsl_db_exists(db, "SELECT 1 FROM blob WHERE length(uuid)>40") + || !fsl_db_exists(db, "SELECT 1 FROM blob WHERE length(uuid)==40")){ + f->cache.hashPolicy = FSL_HPOLICY_SHA3; + }else{ + f->cache.hashPolicy = FSL_HPOLICY_AUTO; + } } } return rc; } } @@ -1036,11 +1043,11 @@ rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, "Could not load UUID for RID %"FSL_ID_T_PFMT, (fsl_id_t)rid); } }else{ - assert(FSL_UUID_STRLEN==fsl_strlen(f->ckout.uuid)); + assert(fsl_is_uuid(f->ckout.uuid)); rc = 0; } f->ckout.rid = rid; }else if(rid==0){ /* This is a legal case not possible before libfossil (and only @@ -1539,10 +1546,58 @@ int fsl_output_f_fsl_cx(void * state, void const * src, fsl_size_t n ){ return (state && src && n) ? fsl_output((fsl_cx*)state, src, n) : (n ? FSL_RC_MISUSE : 0); } + +int fsl_cx_hash_buffer( const fsl_cx * f, int useAlternate, + fsl_buffer const * pIn, fsl_buffer * pOut){ + /* fossil(1) counterpart: hname_hash() */ + if(useAlternate){ + switch(f->cache.hashPolicy){ + case FSL_HPOLICY_AUTO: + case FSL_HPOLICY_SHA1: + return fsl_sha3sum_buffer(pIn, pOut); + case FSL_HPOLICY_SHA3: + return fsl_sha1sum_buffer(pIn, pOut); + default: return FSL_RC_RANGE; + } + }else{ + switch(f->cache.hashPolicy){ + case FSL_HPOLICY_SHA1: + case FSL_HPOLICY_AUTO: + return fsl_sha1sum_buffer(pIn, pOut); + case FSL_HPOLICY_SHA3: + case FSL_HPOLICY_SHA3_ONLY: + case FSL_HPOLICY_SHUN_SHA1: + return fsl_sha3sum_buffer(pIn, pOut); + } + } + assert(!"not reached"); + return FSL_RC_RANGE; +} + +fsl_hash_policy_t fsl_cx_hash_policy_set(fsl_cx *f, fsl_hash_policy_t p){ + fsl_hash_policy_t const old = f->cache.hashPolicy; + f->cache.hashPolicy = p; + return old; +} +fsl_hash_policy_t fsl_cx_hash_policy_get(fsl_cx const*f){ + return f->cache.hashPolicy; +} + +char const * fsl_cx_hash_policy_name(fsl_cx const*f){ + switch(f->cache.hashPolicy){ + case FSL_HPOLICY_SHUN_SHA1: return "shun-sha1"; + case FSL_HPOLICY_SHA3: return "sha3"; + case FSL_HPOLICY_SHA3_ONLY: return "sha3-only"; + case FSL_HPOLICY_SHA1: return "sha1"; + case FSL_HPOLICY_AUTO: return "auto"; + default: return "unknown"; + } +} + #if 0 struct tm * fsl_cx_localtime( fsl_cx const * f, const time_t * clock ){ if(!clock) return NULL; else if(!f) return localtime(clock); Index: src/fsl_md5.c ================================================================== --- src/fsl_md5.c +++ src/fsl_md5.c @@ -344,11 +344,11 @@ if(rc) return rc; else if(read) fsl_md5_update(&ctx, (unsigned char*)zBuf, read); if(read < (fsl_size_t)BufSize) break; } fsl_buffer_reset(pCksum); - rc = fsl_buffer_resize(pCksum, FSL_UUID_STRLEN); + rc = fsl_buffer_resize(pCksum, FSL_MD5_STRLEN); if(!rc){ fsl_md5_final(&ctx, zResult); fsl_md5_digest_to_base16(zResult, fsl_buffer_str(pCksum)); } return rc; Index: src/fsl_mf.c ================================================================== --- src/fsl_mf.c +++ src/fsl_mf.c @@ -74,23 +74,25 @@ } fsl_card_Q * fsl_card_Q_malloc(int type, fsl_uuid_cstr target, fsl_uuid_cstr baseline){ - if(!type || !target || !fsl_is_uuid(target) - || (baseline && !fsl_is_uuid(baseline))) return NULL; + int const targetLen = target ? fsl_is_uuid(target) : 0; + int const baselineLen = baseline ? fsl_is_uuid(baseline) : 0; + if(!type || !target || !targetLen + || (baseline && !baselineLen)) return NULL; else{ fsl_card_Q * c = (fsl_card_Q*)fsl_malloc(sizeof(fsl_card_Q)); if(c){ int rc = 0; *c = fsl_card_Q_empty; c->type = type; - c->target = fsl_strndup(target, FSL_UUID_STRLEN); + c->target = fsl_strndup(target, targetLen); if(!c->target) rc = FSL_RC_OOM; else if(baseline){ - c->baseline = fsl_strndup( baseline, FSL_UUID_STRLEN); + c->baseline = fsl_strndup(baseline, baselineLen); if(!c->baseline) rc = FSL_RC_OOM; } if(rc){ fsl_card_Q_free(c); c = NULL; @@ -149,21 +151,22 @@ fsl_card_F * fsl_card_F_malloc(char const * name, char const * uuid, fsl_file_perm_t perm, char const * oldName){ fsl_card_F * t; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; if(!name || !*name) return NULL; - else if(uuid && !fsl_is_uuid(uuid)) return NULL; + else if(uuid && !uLen) return NULL; t = (fsl_card_F *)fsl_malloc(sizeof(fsl_card_F)); if(t){ int rc = 0; *t = fsl_card_F_empty; t->perm = perm; t->name = fsl_strdup(name); if(!t->name) rc = FSL_RC_OOM; - if(!rc && uuid){ - t->uuid = fsl_strdup(uuid); + if(!rc && uLen){ + t->uuid = fsl_strndup(uuid, uLen); if(!t->uuid) rc = FSL_RC_OOM; } if(!rc && oldName){ t->priorName = fsl_strdup(oldName); if(!t->priorName) rc = FSL_RC_OOM; @@ -531,30 +534,32 @@ is assumed to be either an SHA1 or MD5 hash value and it is validated against fsl_validate16(value,valLen), returning FSL_RC_CA_SYNTAX if that check fails. */ static int fsl_deck_sethex_impl( fsl_deck * mf, fsl_uuid_cstr value, - char letter, - fsl_size_t assertLen, - char ** mfMember ){ + char letter, + fsl_size_t assertLen, + char ** mfMember ){ assert(mf); - assert( assertLen==FSL_UUID_STRLEN || assertLen==FSL_MD5_STRLEN ); + assert( assertLen==FSL_UUIDv1_STRLEN + || assertLen==FSL_UUIDv2_STRLEN + || assertLen==FSL_MD5_STRLEN ); if(!fsl_deck_check_type(mf,letter)) return FSL_RC_TYPE; else if(!value){ fsl_deck_free_string(mf, *mfMember); *mfMember = NULL; return 0; - }else if(fsl_strlen(value) != (assertLen)){ + }else if(fsl_strlen(value) != assertLen){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid length for %c-card: expecting %d.", letter, (int)assertLen); }else if(!fsl_validate16(value, assertLen)) { return fsl_error_set(&mf->error, FSL_RC_CA_SYNTAX, "Invalid hexadecimal value for %c-card.", letter); }else{ fsl_deck_free_string(mf, *mfMember); - *mfMember = fsl_strndup( value, assertLen ); + *mfMember = fsl_strndup(value, assertLen); return *mfMember ? 0 : FSL_RC_OOM; } } /** @@ -576,16 +581,21 @@ } int fsl_deck_B_set( fsl_deck * mf, fsl_uuid_cstr uuidBaseline){ if(!mf) return FSL_RC_MISUSE; else{ + int const bLen = fsl_is_uuid(uuidBaseline); + if(!bLen){ + return fsl_error_set(&mf->error, FSL_RC_CA_SYNTAX, + "Invalid B-card value: %s", uuidBaseline); + } if(mf->B.baseline){ fsl_deck_finalize(mf->B.baseline); mf->B.baseline = NULL; } return fsl_deck_sethex_impl(mf, uuidBaseline, 'B', - FSL_UUID_STRLEN, &mf->B.uuid); + bLen, &mf->B.uuid); } } /** Internal impl for card setters which consider of a simple (char *) @@ -621,12 +631,13 @@ } } int fsl_deck_K_set( fsl_deck * mf, fsl_uuid_cstr uuid){ - return mf - ? fsl_deck_sethex_impl(mf, uuid, 'K', FSL_UUID_STRLEN, &mf->K) + int const uLen = fsl_is_uuid(uuid); + return (mf && uLen) + ? fsl_deck_sethex_impl(mf, uuid, 'K', uLen, &mf->K) : FSL_RC_MISUSE; } int fsl_deck_L_set( fsl_deck * mf, char const * v, fsl_int_t n){ return mf @@ -635,14 +646,15 @@ } int fsl_deck_M_add( fsl_deck * mf, char const *uuid){ int rc; char * dupe; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; if(!mf || !uuid) return FSL_RC_MISUSE; else if(!fsl_deck_check_type(mf, 'M')) return FSL_RC_TYPE; - else if(!fsl_is_uuid(uuid)) return FSL_RC_RANGE; - dupe = fsl_strndup(uuid, FSL_UUID_STRLEN); + else if(!uLen) return FSL_RC_RANGE; + dupe = fsl_strndup(uuid, uLen); if(!dupe) rc = FSL_RC_OOM; else{ rc = fsl_list_append( &mf->M, dupe ); if(rc){ fsl_free(dupe); @@ -660,17 +672,18 @@ int fsl_deck_P_add( fsl_deck * mf, char const *parentUuid){ int rc; char * dupe; + int const uLen = parentUuid ? fsl_is_uuid(parentUuid) : 0; if(!mf || !parentUuid || !*parentUuid) return FSL_RC_MISUSE; else if(!fsl_deck_check_type(mf, 'P')) return FSL_RC_TYPE; - else if(!fsl_is_uuid(parentUuid)){ + else if(!uLen){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid UUID for P-card."); } - dupe = fsl_strndup(parentUuid, FSL_UUID_STRLEN); + dupe = fsl_strndup(parentUuid, uLen); if(!dupe) rc = FSL_RC_OOM; else{ rc = fsl_list_append( &mf->P, dupe ); if(rc){ fsl_free(dupe); @@ -868,20 +881,21 @@ } int fsl_deck_A_set( fsl_deck * mf, char const * name, char const * tgt, char const * uuidSrc ){ + int const uLen = (uuidSrc && *uuidSrc) ? fsl_is_uuid(uuidSrc) : 0; if(!mf || !name || !tgt) return FSL_RC_MISUSE; else if(!fsl_deck_check_type(mf, 'A')) return FSL_RC_TYPE; else if(!*tgt){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid target name in A card."); } /* TODO: validate tgt based on mf->type and require UUID for types EVENT/TICKET. */ - else if(uuidSrc && *uuidSrc && !fsl_is_uuid(uuidSrc)){ + else if(uuidSrc && *uuidSrc && !uLen){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid source UUID in A card."); } else{ int rc = 0; @@ -889,12 +903,12 @@ fsl_deck_free_string(mf, mf->A.src); fsl_deck_free_string(mf, mf->A.name); mf->A.name = mf->A.src = NULL; if(! (mf->A.tgt = fsl_strdup(tgt))) rc = FSL_RC_OOM; else if( !(mf->A.name = fsl_strdup(name))) rc = FSL_RC_OOM; - else if(uuidSrc && *uuidSrc){ - mf->A.src = fsl_strndup(uuidSrc,FSL_UUID_STRLEN); + else if(uLen){ + mf->A.src = fsl_strndup(uuidSrc,uLen); if(!mf->A.src) rc = FSL_RC_OOM /* Leave mf->A.tgt/name for downstream cleanup. */; } return rc; } @@ -910,22 +924,23 @@ return 0; } } int fsl_deck_E_set( fsl_deck * mf, fsl_double_t date, char const * uuid){ - if(!mf || !uuid) return FSL_RC_MISUSE; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!mf || !uLen) return FSL_RC_MISUSE; else if(date<=0){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid date value for E card."); - }else if(!fsl_is_uuid(uuid)){ + }else if(!uLen){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid UUID for E card."); } else{ mf->E.julian = date; fsl_deck_free_string(mf, mf->E.uuid); - mf->E.uuid = fsl_strndup(uuid, FSL_UUID_STRLEN); + mf->E.uuid = fsl_strndup(uuid, uLen); return mf->E.uuid ? 0 : FSL_RC_OOM; } } int fsl_deck_F_add2( fsl_deck * mf, fsl_card_F * t){ @@ -948,10 +963,11 @@ int fsl_deck_F_add( fsl_deck * mf, char const * name, char const * uuid, fsl_file_perm_t perms, char const * oldName){ + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; if(!mf || !name) return FSL_RC_MISUSE; else if(!fsl_deck_check_type(mf, 'F')) return FSL_RC_TYPE; else if(!*name){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "F-card name may not be empty."); @@ -960,11 +976,11 @@ || (oldName && !fsl_is_simple_pathname(oldName, 1))){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid filename for F-card (simple form required): " "name=[%s], oldName=[%s].", name, oldName); } - else if(uuid && !fsl_is_uuid(uuid)){ + else if(uuid && !uLen){ return fsl_error_set(&mf->error, FSL_RC_RANGE, "Invalid UUID for F-card."); } else { int rc; @@ -1467,23 +1483,20 @@ set to 0 before running the visit iteration. */ static int fsl_list_v_mf_output_card_P(void * obj, void * visitorState ){ fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; char const * uuid = (char const *)obj; - fsl_size_t len; - assert(uuid); - if(!fsl_is_uuid(uuid)){ + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!uLen){ os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, "Invalid UUID in P card."); } else if(!os->counter++) fsl_appendf_f_mf(os, "P ", 2); else fsl_appendf_f_mf(os, " ", 1); /* Reminder: fsl_appendf_f_mf() updates os->rc. */ if(!os->rc){ - len = (fsl_size_t)fsl_strlen(uuid); - assert(FSL_UUID_STRLEN==len) /* is enforced by fsl_mf_add_P() */; - fsl_appendf_f_mf(os, uuid, len); + fsl_appendf_f_mf(os, uuid, (fsl_size_t)uLen); } return os->rc; } @@ -2959,11 +2972,11 @@ fsl_buffer_clear(&buf); if(rc) goto end; }/*WIKI*/ if( d->type==FSL_CATYPE_EVENT ){ - char buf[FSL_UUID_STRLEN + 7 /* event-UUID */] = {0}; + char buf[FSL_UUIDv2_STRLEN + 7 /* event-UUID */] = {0}; char zLength[40] = {0}; fsl_id_t tagid; fsl_id_t prior, subsequent; char const * zWiki; char const * zTag; @@ -3624,15 +3637,15 @@ /* Parsing helpers... */ #define TOKEN(DEFOS) tokLen=0; token = fsl_mf_next_token(&x,&tokLen); \ if(token && tokLen && (DEFOS)) fsl_bytes_defossilize(token, &tokLen) #define TOKEN_EXISTS(MSG_IF_NOT) if(!token){ SYNTAX(MSG_IF_NOT); }(void)0 -#define TOKEN_CHECKHEX(LEN,MSG) if(token \ - && ((LEN)!=tokLen || \ - !fsl_validate16((char const *)token,(LEN)))){ \ - SYNTAX(MSG); } -#define TOKEN_UUID(CARD) TOKEN_CHECKHEX(FSL_UUID_STRLEN,"Malformed UUID in " #CARD "-card") +#define TOKEN_CHECKHEX(MSG) if(token && (int)tokLen!=fsl_is_uuid((char const *)token))\ + { SYNTAX(MSG); } +#define TOKEN_UUID(CARD) TOKEN_CHECKHEX("Malformed UUID in " #CARD "-card") +#define TOKEN_MD5(ERRMSG) if(!token || FSL_MD5_STRLEN!=(int)tokLen) \ + {SYNTAX(ERRMSG);} /** Reminder: we do not know the type of the manifest at this point, so all of the fsl_deck_add/set() bits below can't do their validation. We have to determine at parse-time (or afterwards) which type of deck it is based on the cards we've seen. We guess @@ -4001,11 +4014,11 @@ if(1R = (char *)token; ++stealBuf; /* rc = fsl_deck_R_set(d, (char const *)token); */ break; } @@ -4030,11 +4043,11 @@ TOKEN(0); TOKEN_EXISTS("Missing name for T-card"); name = token; TOKEN(0); TOKEN_EXISTS("Missing UUID on T-card"); - if(FSL_UUID_STRLEN==tokLen){ + if(fsl_is_uuid_len((int)tokLen)){ TOKEN_UUID(T); /* A valid UUID */ if(FSL_CATYPE_EVENT==d->type){ SYNTAX("Non-self-referential T-card in Event artifact"); } @@ -4247,10 +4260,14 @@ "Invalid RID for fsl_deck_load_rid(): " "%"FSL_ID_T_PFMT, (fsl_id_t)rid); } rc = fsl_content_get(f, rid, &buf); if(!rc){ +#if 0 + MARKER(("fsl_content_get(%d) len=%d =\n%.*s\n", + (int)rid, (int)buf.used, (int)buf.used, (char const*)buf.mem)); +#endif fsl_deck_clean(d); fsl_deck_init(f, d, FSL_CATYPE_ANY); #if 0 /* If we set d->type=type, the parser can fail more @@ -4261,10 +4278,13 @@ */ d->type = type /* may help parsing fail more quickly if it's not the type we want.*/; #endif rc = fsl_deck_parse(d, &buf); +#if 0 + MARKER(("rid=%d, d->rid=%d\n", (int)rid, (int)d->rid)); +#endif if(!rc){ assert(rid == d->rid); if( type!=FSL_CATYPE_ANY && d->type!=type ){ rc = fsl_cx_err_set(f, FSL_RC_TYPE, "RID %"FSL_ID_T_PFMT" is of type %s, " Index: src/fsl_repo.c ================================================================== --- src/fsl_repo.c +++ src/fsl_repo.c @@ -149,16 +149,16 @@ fsl_stmt_finalize(&q); goto gotit; } symLen = fsl_strlen(sym); - /* SHA1 hash or prefix */ + /* SHA1/SHA3 hash or prefix */ if( symLen>=4 - && symLen<=FSL_UUID_STRLEN + && symLen<=FSL_UUIDv2_STRLEN && fsl_validate16(sym, symLen) ){ fsl_stmt q = fsl_stmt_empty; - char zUuid[FSL_UUID_STRLEN+1]; + char zUuid[FSL_UUIDv2_STRLEN+1]; memcpy(zUuid, sym, symLen); zUuid[symLen] = 0; fsl_canonical16(zUuid, symLen); rid = 0; /* Reminder to self: caching these queries would be cool but it @@ -298,32 +298,38 @@ } return rc; } fsl_id_t fsl_uuid_to_rid( fsl_cx * f, char const * uuid ){ - fsl_db * db = fsl_needs_repo(f); - fsl_size_t uuidLen = (uuid && db) ? fsl_strlen(uuid) : 0; - if(!f || !uuid) return -1; + fsl_db * const db = fsl_needs_repo(f); + fsl_size_t const uuidLen = (uuid && db) ? fsl_strlen(uuid) : 0; + if(!f || !uuid || !uuidLen) return -1; else if(!db){ /* f's error state has already been set */ return -2; } else if(!fsl_validate16(uuid, uuidLen)){ fsl_cx_err_set(f, FSL_RC_RANGE, "Invalid UUID (prefix): %s", uuid); return -3; } - else if(uuidLen>FSL_UUID_STRLEN){ + else if(uuidLen>FSL_UUIDv2_STRLEN){ fsl_cx_err_set(f, FSL_RC_RANGE, "UUID is too long: %s", uuid); return -4; } else { fsl_id_t rid = -5; fsl_stmt q = fsl_stmt_empty; fsl_stmt * qS = NULL; int rc; - rc = (uuidLen==FSL_UUID_STRLEN) - /* Optimization for the common internally-used case */ + rc = fsl_is_uuid_len((int)uuidLen) + /* Optimization for the common internally-used case. + + FIXME: there is an *astronomically small* chance of a prefix + collision on a v1-length uuidLen against a v2-length + blob.uuid value, leading to no match found for an existing v2 + uuid here. Like... a *REALLY* small chance. + */ ? fsl_db_prepare_cached(db, &qS, "SELECT rid FROM blob WHERE " "uuid=?") : fsl_db_prepare(db, &q, "SELECT rid FROM blob WHERE " @@ -330,11 +336,11 @@ "uuid GLOB '%s*'", uuid); if(!rc){ fsl_stmt * st = qS ? qS : &q; if(qS){ - rc = fsl_stmt_bind_text(qS, 1, uuid, FSL_UUID_STRLEN, 0); + rc = fsl_stmt_bind_text(qS, 1, uuid, (fsl_int_t)uuidLen, 0); } if(!rc){ rc = fsl_stmt_step(st); switch(rc){ case FSL_RC_STEP_ROW: Index: src/fsl_sha1.c ================================================================== --- src/fsl_sha1.c +++ src/fsl_sha1.c @@ -1730,11 +1730,11 @@ */ void fsl_sha1_digest_to_base16(unsigned char *digest, char *zBuf){ static char const zEncode[] = "0123456789abcdef"; int ix; - for(ix=0; ix<20; ix++){ + for(ix=0; ix>4)&0xf]; *zBuf++ = zEncode[*digest++ & 0xf]; } *zBuf = '\0'; } @@ -1801,16 +1801,22 @@ } } return 0; #endif } + +char const * fsl_sha1_final_hex(fsl_sha1_cx *context, char * zHex){ + unsigned char zResult[FSL_UUIDv1_STRLEN/2]; + fsl_sha1_final(context, zResult); + fsl_sha1_digest_to_base16(zResult, zHex); + return (char const *)zHex; +} int fsl_sha1sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum){ + enum { BufSize = 1024 * 4 }; fsl_sha1_cx ctx; int rc; - unsigned char zResult[20]; - enum { BufSize = 1024 * 4 }; unsigned char zBuf[BufSize]; if(!src || !pCksum) return FSL_RC_MISUSE; fsl_sha1_init(&ctx); for(;;){ fsl_size_t read = (fsl_size_t)BufSize; @@ -1818,14 +1824,13 @@ if(rc) return rc; else if(read) fsl_sha1_update(&ctx, (unsigned char*)zBuf, read); if(read < (fsl_size_t)BufSize) break; } fsl_buffer_reset(pCksum); - rc = fsl_buffer_resize(pCksum, FSL_UUID_STRLEN); + rc = fsl_buffer_resize(pCksum, FSL_UUIDv1_STRLEN); if(!rc){ - fsl_sha1_final(&ctx, zResult); - fsl_sha1_digest_to_base16(zResult, fsl_buffer_str(pCksum)); + fsl_sha1_final_hex(&ctx, fsl_buffer_str(pCksum)); } return rc; } int fsl_sha1sum_filename(const char *zFilename, fsl_buffer *pCksum){ @@ -1842,11 +1847,11 @@ return rc; #else /* Requires v1 code which has not yet been ported in. */ FILE *in; fsl_sha1_cx ctx; - unsigned char zResult[20]; + unsigned char zResult[FSL_UUIDv1_STRLEN/2]; char zBuf[10240]; if( fsl_wd_islink(zFilename) ){ /* Instead of file content, return sha1 of link destination path */ Blob destinationPath; @@ -1869,11 +1874,11 @@ if( n<=0 ) break; fsl_sha1_update(&ctx, (unsigned char*)zBuf, (unsigned)n); } fclose_fclose(in); blob_zero(pCksum); - blob_resize(pCksum, FSL_UUID_STRLEN); + blob_resize(pCksum, FSL_UUIDv1_STRLEN); fsl_sha1_final(&ctx, zResult); fsl_sha1_digest_to_base16(zResult, blob_buffer(pCksum)); return 0; #endif } @@ -1881,20 +1886,18 @@ int fsl_sha1sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum){ if(!pIn || !pCksum) return FSL_RC_MISUSE; else{ fsl_sha1_cx ctx; - unsigned char zResult[20]; int rc; fsl_sha1_init(&ctx); fsl_sha1_update(&ctx, pIn->mem, pIn->used); fsl_buffer_reset(pCksum); - rc = fsl_buffer_resize(pCksum, FSL_UUID_STRLEN + rc = fsl_buffer_resize(pCksum, FSL_UUIDv1_STRLEN /*resize() adds 1 for NUL*/); if(!rc){ - fsl_sha1_final(&ctx, zResult); - fsl_sha1_digest_to_base16(zResult, fsl_buffer_str(pCksum)); + fsl_sha1_final_hex(&ctx, fsl_buffer_str(pCksum)); assert(0==pCksum->mem[pCksum->used]); } return rc; } } @@ -1901,16 +1904,14 @@ char *fsl_sha1sum_cstr(const char *zIn, fsl_int_t len){ if(!zIn || !len) return NULL; else{ fsl_sha1_cx ctx; - unsigned char zResult[20]; - char * zDigest = (char *)fsl_malloc(FSL_UUID_STRLEN+1); - if(!zDigest) return NULL; + char * zHex = (char *)fsl_malloc(FSL_UUIDv1_STRLEN+1); + if(!zHex) return NULL; fsl_sha1_init(&ctx); fsl_sha1_update(&ctx, zIn, (len<0) ? fsl_strlen(zIn) : (fsl_size_t)len); - fsl_sha1_final(&ctx, zResult); - fsl_sha1_digest_to_base16(zResult, zDigest); - return zDigest; + fsl_sha1_final_hex(&ctx, zHex); + return zHex; } } Index: src/fsl_sha3.c ================================================================== --- src/fsl_sha3.c +++ src/fsl_sha3.c @@ -33,10 +33,18 @@ #include /* strlen() */ #include /* NULL on linux */ #include #include +#if 0 +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) +#endif + /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. ** ** For best performance, an attempt is made to guess at the byte-order @@ -414,11 +422,15 @@ case 512: return FSL_SHA3_512; default: return FSL_SHA3_INVALID; } } -void fsl_sha3_init(fsl_sha3_cx *p, enum fsl_sha3_hash_size iSize){ +void fsl_sha3_init(fsl_sha3_cx *cx){ + fsl_sha3_init2(cx, FSL_SHA3_DEFAULT); +} +void fsl_sha3_init2(fsl_sha3_cx *p, enum fsl_sha3_hash_size iSize){ + assert(iSize>0); memset(p, 0, sizeof(*p)); p->size = iSize; if( iSize>=128 && iSize<=512 ){ p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; }else{ @@ -507,7 +519,128 @@ DigestToBase16( &p->u.x[p->nRate], p->hex, (int)p->size/8 ); assert(0 == p->hex[(int)p->size/4+1]); return &p->u.x[p->nRate]; } +void fsl_sha3_digest_to_base16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int ix; + for(ix=0; ix>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + + +int fsl_sha3sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum){ + fsl_sha3_cx ctx; + int rc; + enum { BufSize = 1024 * 4 }; + unsigned char zBuf[BufSize]; + if(!src || !pCksum) return FSL_RC_MISUSE; + fsl_sha3_init(&ctx); + for(;;){ + fsl_size_t read = (fsl_size_t)BufSize; + rc = src(srcState, zBuf, &read); + if(rc) return rc; + else if(read) fsl_sha3_update(&ctx, (unsigned char*)zBuf, read); + if(read < (fsl_size_t)BufSize) break; + } + fsl_buffer_reset(pCksum); + rc = fsl_buffer_resize(pCksum, FSL_UUIDv2_STRLEN); + if(!rc){ + fsl_sha3_end(&ctx); + rc = fsl_buffer_append(pCksum, ctx.hex, fsl_strlen((const char *)ctx.hex)); + } + return rc; +} + +int fsl_sha3sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum){ + if(!pIn || !pCksum) return FSL_RC_MISUSE; + else{ + fsl_sha3_cx ctx; + int rc; + fsl_sha3_init(&ctx); + fsl_sha3_update(&ctx, pIn->mem, pIn->used); + fsl_buffer_reset(pCksum); + rc = fsl_buffer_resize(pCksum, FSL_UUIDv2_STRLEN + /*resize() adds 1 for NUL*/); + if(!rc){ + pCksum->used = 0; + fsl_sha3_end(&ctx); + assert(fsl_strlen((char const*)ctx.hex)==FSL_UUIDv2_STRLEN); + rc = fsl_buffer_append(pCksum, ctx.hex, fsl_strlen((char const*)ctx.hex)); + assert(!rc && "Cannot fail - pre-allocated"); + if(!rc){ + assert(0==pCksum->mem[pCksum->used]); + } + } + return rc; + } +} + +char *fsl_sha3sum_cstr(const char *zIn, fsl_int_t len){ + if(!zIn || !len) return NULL; + else{ + fsl_sha3_cx ctx; + fsl_sha3_init(&ctx); + fsl_sha3_update(&ctx, zIn, + (len<0) ? fsl_strlen(zIn) : (fsl_size_t)len); + fsl_sha3_end(&ctx); + return fsl_strdup((char const *)ctx.hex); + } +} + +int fsl_sha3sum_filename(const char *zFilename, fsl_buffer *pCksum){ + if(!zFilename || !pCksum) return FSL_RC_MISUSE; + else{ +#if 1 + int rc; + FILE *in = fsl_fopen(zFilename, "rb"); + if(!in) rc = FSL_RC_IO; + else{ + rc = fsl_sha3sum_stream(fsl_input_f_FILE, in, pCksum); + fsl_fclose(in); + } + return rc; +#else + /* Requires v1 code which has not yet been ported in. */ + FILE *in; + fsl_sha1_cx ctx; + char zBuf[10240]; + int rc; + + if( fsl_wd_islink(zFilename) ){ + /* Instead of file content, return sha3 of link destination path */ + Blob destinationPath; + + blob_read_link(&destinationPath, zFilename); + rc = fsl_sha3sum_buffer(&destinationPath, pCksum); + fsl_buffer_clear(&destinationPath); + return rc; + } + + in = fossil_fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + fsl_sha3_init(&ctx); + for(;;){ + int n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + fsl_sha3_update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose_fclose(in); + blob_zero(pCksum); + blob_resize(pCksum, FSL_UUIDv1_STRLEN); + fsl_sha3_end(&ctx); + rc = fsl_buffer_append(pCksum, ctx.hex, fsl_strlen(ctx.hex)); + return rc; +#endif + } +} + #undef SHA3_BYTEORDER +#undef MARKER Index: src/fsl_tag.c ================================================================== --- src/fsl_tag.c +++ src/fsl_tag.c @@ -51,18 +51,19 @@ fsl_card_T * fsl_card_T_malloc(fsl_tag_type tagType, char const * uuid, char const * name, char const * value){ fsl_card_T * t; - if(uuid && !fsl_is_uuid(uuid)) return NULL; + int const uuidLen = uuid ? fsl_is_uuid(uuid) : 0; + if(uuid && !uuidLen) return NULL; t = (fsl_card_T *)fsl_malloc(sizeof(fsl_card_T)); if(t){ int rc = 0; *t = fsl_card_T_empty; t->type = tagType; if(uuid && *uuid){ - t->uuid = fsl_strndup(uuid, FSL_UUID_STRLEN); + t->uuid = fsl_strndup(uuid, uuidLen); if(!t->uuid) rc = FSL_RC_OOM; } if(!rc && name && *name){ t->name = fsl_strdup(name); if(!t->name){ Index: src/fsl_vfile.c ================================================================== --- src/fsl_vfile.c +++ src/fsl_vfile.c @@ -86,11 +86,11 @@ while( !rc && !(rc=fsl_deck_F_next(&d, &fc)) && fc){ fsl_id_t rid; fsl_int64_t size; if(!fc->uuid /* was removed in this version */ || fsl_uuid_is_shunned(f,fc->uuid)) continue; - rc = fsl_stmt_bind_text(&qRid, 1, fc->uuid, FSL_UUID_STRLEN, 0); + rc = fsl_stmt_bind_text(&qRid, 1, fc->uuid, -1, 0); if(rc) break; rc = fsl_stmt_step(&qRid); if(FSL_RC_STEP_ROW==rc){ rid = fsl_stmt_g_id(&qRid,0); size = fsl_stmt_g_int64(&qRid,1); @@ -137,10 +137,41 @@ if(dbC->error.code) fsl_cx_uplift_db_error(f, dbC); else if(dbR->error.code) fsl_cx_uplift_db_error(f, dbR); } return rc; } +/** + Internal code de-duplifier for places which need to re-check a + file's hash in order to be sure whether it was really + modified. hashLen must be the length of the previous (db-side) + hash of the file. This routine will hash that file using the same + hash type. + + Returns 0 on success. On error, if *errReported is set to non-0 then the + error state has already been set +*/ +static int fsl_vfile_recheck_file_hash( fsl_cx * f, const char * zName, + int hashLen, fsl_buffer * pTgt ){ + int errReported = 0; + int rc = 0; + pTgt->used = 0; + if(FSL_UUIDv1_STRLEN==hashLen){ + rc = fsl_sha1sum_filename(zName, pTgt); + }else if(FSL_UUIDv2_STRLEN==hashLen){ + rc = fsl_sha3sum_filename(zName, pTgt); + }else{ + rc = fsl_cx_err_set(f, FSL_RC_CHECKSUM_MISMATCH, + "Cannot determine which hash to use for file: %s", + zName); + errReported = 1; + } + if(rc && !errReported && FSL_RC_OOM != rc){ + rc = fsl_cx_err_set(f, rc, "Error %s while reading SHA of file: %s", + fsl_rc_cstr(rc), zName); + } + return rc; +} /** UNTESTED single-file version of fsl_vfile_changes_scan(). vid is the version to scan changes against. If 0 or less, the @@ -218,12 +249,10 @@ goto end; } { - static char const * errReadingSha1 = - "Error %s while reading SHA1 of file: %s"; fsl_id_t id = fsl_stmt_g_id(q, 0); fsl_id_t rid = fsl_stmt_g_id(q, 2); /* fsl_size_t nName = 0; */ char const * zName = fsl_stmt_g_text(q, 1, NULL/* &nName */); /* fsl_size_t const rootLen = fsl_strlen(f->ckout.dir); */ @@ -298,29 +327,22 @@ need to check the mtime or sha1sum */ changed = FSL_VFILE_CHANGE_MOD; }else if( !forceVFileUpdate && changed==FSL_VFILE_CHANGE_MOD && rid!=0 && !isDeleted ){ - /* File is believed to have changed but it is the same size. + /* File is believed to have changed but it is the same size. Double check that it really has changed by looking at content. */ fsl_size_t nUuid = 0; char const * uuid; fsl_buffer * fileCksum = &f->fileContent; assert(!fileCksum->used && "Misuse of f->fileContent."); assert( origSize==currentSize ); - fileCksum->used = 0; - rc = fsl_sha1sum_filename(zName, fileCksum); - if(rc){ - if(FSL_RC_OOM != rc){ - fsl_cx_err_set(f, rc, errReadingSha1, - fsl_rc_cstr(rc), zName); - } - goto end; - } - assert(FSL_UUID_STRLEN==fileCksum->used); uuid = fsl_stmt_g_text(q, 5, &nUuid); - assert(uuid && (FSL_UUID_STRLEN==nUuid)); + assert(uuid && fsl_is_uuid(uuid)==(int)nUuid); + rc = fsl_vfile_recheck_file_hash(f, zName, (int)nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); if( 0 == fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ changed = 0; } fileCksum->used = 0; /* MARKER(("SHA1 compare says %d: %s\n", changed, zName)); */ @@ -330,28 +352,21 @@ || changed==FSL_VFILE_CHANGE_INTEGRATE_MOD) && (useMtime==0 || currentMtime!=oldMtime) ){ /* For files that were formerly believed to be unchanged or that were changed by merging, if their mtime changes, or unconditionally if --sha1sum is used, check to see if they have been edited by - looking at their SHA1 sum */ + looking at their hash */ fsl_size_t nUuid = 0; char const * uuid; fsl_buffer * fileCksum = &f->fileContent; assert(!fileCksum->used && "Misuse of f->fileContent."); assert( origSize==currentSize ); - fileCksum->used = 0; - rc = fsl_sha1sum_filename(zName, fileCksum); - if(rc){ - if(FSL_RC_OOM != rc){ - fsl_cx_err_set(f, rc, errReadingSha1, - fsl_rc_cstr(rc), zName); - } - goto end; - } - assert(FSL_UUID_STRLEN==fileCksum->used); uuid = fsl_stmt_g_text(q, 5, &nUuid); - assert(uuid && (FSL_UUID_STRLEN==nUuid)); + assert(uuid && fsl_is_uuid(uuid)==(int)nUuid); + rc = fsl_vfile_recheck_file_hash(f, zName, (int)nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); if( fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ changed = FSL_VFILE_CHANGE_MOD; } fileCksum->used = 0; /* MARKER(("SHA1 compare says %d: %s\n", changed, zName)); */ @@ -408,12 +423,10 @@ fsl_deck deck = fsl_deck_empty; #ifndef _WIN32 fsl_deck * d = NULL; #endif fsl_size_t rootLen; - static char const * errReadingSha1 = - "Error %s while reading SHA1 of file: %s"; if(!db) return FSL_RC_NOT_A_CHECKOUT; assert(f->ckout.dir); if(vid<0) vid = f->ckout.rid; @@ -541,21 +554,15 @@ fsl_size_t nUuid = 0; char const * uuid; fsl_buffer * fileCksum = &f->fileContent; assert(!fileCksum->used && "Misuse of f->fileContent."); assert( origSize==currentSize ); - rc = fsl_sha1sum_filename(zName, fileCksum); - if(rc){ - if(FSL_RC_OOM != rc){ - fsl_cx_err_set(f, rc, errReadingSha1, - fsl_rc_cstr(rc), zName); - } - goto end; - } - assert(FSL_UUID_STRLEN==fileCksum->used); uuid = fsl_stmt_g_text(&q, 5, &nUuid); - assert(uuid && (FSL_UUID_STRLEN==nUuid)); + assert(uuid && fsl_is_uuid_len((int)nUuid)); + rc = fsl_vfile_recheck_file_hash(f, zName, (int)nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); if( 0 == fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ changed = 0; } fileCksum->used = 0; /* MARKER(("SHA1 compare says %d: %s\n", changed, zName)); */ @@ -571,22 +578,15 @@ fsl_size_t nUuid = 0; char const * uuid; fsl_buffer * fileCksum = &f->fileContent; assert(!fileCksum->used && "Misuse of f->fileContent."); assert( origSize==currentSize ); - fileCksum->used = 0; - rc = fsl_sha1sum_filename(zName, fileCksum); - if(rc){ - if(FSL_RC_OOM != rc){ - fsl_cx_err_set(f, rc, errReadingSha1, - fsl_rc_cstr(rc), zName); - } - goto end; - } - assert(FSL_UUID_STRLEN==fileCksum->used); uuid = fsl_stmt_g_text(&q, 5, &nUuid); - assert(uuid && (FSL_UUID_STRLEN==nUuid)); + assert(uuid && fsl_is_uuid_len((int)nUuid)); + rc = fsl_vfile_recheck_file_hash(f, zName, (int)nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); if( fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ changed = FSL_VFILE_CHANGE_MOD; } fileCksum->used = 0; /* MARKER(("SHA1 compare says %d: %s\n", changed, zName)); */