Index: src/db.c ================================================================== --- src/db.c +++ src/db.c @@ -547,12 +547,13 @@ va_end(ap); z = blob_str(&sql); while( rc==SQLITE_OK && z[0] ){ pStmt = 0; rc = sqlite3_prepare_v2(g.db, z, -1, &pStmt, &zEnd); - if( rc!=SQLITE_OK ) break; - if( pStmt ){ + if( rc ){ + db_err("%s: {%s}", sqlite3_errmsg(g.db), z); + }else if( pStmt ){ db.nPrepare++; while( sqlite3_step(pStmt)==SQLITE_ROW ){} rc = sqlite3_finalize(pStmt); if( rc ) db_err("%s: {%.*s}", sqlite3_errmsg(g.db), (int)(zEnd-z), z); } @@ -1380,15 +1381,10 @@ while( db.pAllStmt ){ db_finalize(db.pAllStmt); } db_end_transaction(1); pStmt = 0; - if( reportErrors ){ - while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){ - fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt)); - } - } db_close_config(); /* If the localdb (the check-out database) is open and if it has ** a lot of unused free space, then VACUUM it as we shut down. */ @@ -1399,12 +1395,18 @@ db_multi_exec("VACUUM;"); } } if( g.db ){ + int rc; sqlite3_wal_checkpoint(g.db, 0); - sqlite3_close(g.db); + rc = sqlite3_close(g.db); + if( rc==SQLITE_BUSY && reportErrors ){ + while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){ + fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt)); + } + } g.db = 0; g.zMainDbType = 0; } g.repositoryOpen = 0; g.localOpen = 0; Index: src/main.mk ================================================================== --- src/main.mk +++ src/main.mk @@ -449,11 +449,12 @@ -DSQLITE_OMIT_LOAD_EXTENSION=1 \ -DSQLITE_ENABLE_LOCKING_STYLE=0 \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_DEFAULT_FILE_FORMAT=4 \ -DSQLITE_OMIT_DEPRECATED \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS4 # Setup the options used to compile the included SQLite shell. SHELL_OPTIONS = -Dmain=sqlite3_shell \ -DSQLITE_OMIT_LOAD_EXTENSION=1 \ -DUSE_SYSTEM_SQLITE=$(USE_SYSTEM_SQLITE) \ Index: src/makemake.tcl ================================================================== --- src/makemake.tcl +++ src/makemake.tcl @@ -157,10 +157,11 @@ -DSQLITE_ENABLE_LOCKING_STYLE=0 -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_OMIT_DEPRECATED -DSQLITE_ENABLE_EXPLAIN_COMMENTS + -DSQLITE_ENABLE_FTS4 } #lappend SQLITE_OPTIONS -DSQLITE_ENABLE_FTS3=1 #lappend SQLITE_OPTIONS -DSQLITE_ENABLE_STAT4 #lappend SQLITE_OPTIONS -DSQLITE_WIN32_NO_ANSI #lappend SQLITE_OPTIONS -DSQLITE_WINNT_MAX_PATH_CHARS=4096 Index: src/manifest.c ================================================================== --- src/manifest.c +++ src/manifest.c @@ -1829,10 +1829,11 @@ for(i=0; inFile; i++){ add_one_mlink(0, 0, rid, p->aFile[i].zUuid, p->aFile[i].zName, 0, isPublic, 1, manifest_file_mperm(&p->aFile[i])); } } + search_doc_touch('c', rid, 0); db_multi_exec( "REPLACE INTO event(type,mtime,objid,user,comment," "bgcolor,euser,ecomment,omtime)" "VALUES('ci'," " coalesce(" @@ -1934,10 +1935,11 @@ if( nWiki>0 ){ zComment = mprintf("Changes to wiki page [%h]", p->zWikiTitle); }else{ zComment = mprintf("Deleted wiki page [%h]", p->zWikiTitle); } + search_doc_touch('w',rid,p->zWikiTitle); db_multi_exec( "REPLACE INTO event(type,mtime,objid,user,comment," " bgcolor,euser,ecomment)" "VALUES('w',%.17g,%d,%Q,%Q," " (SELECT value FROM tagxref WHERE tagid=%d AND rid=%d AND tagtype>1)," @@ -1986,10 +1988,11 @@ } } if( subsequent ){ content_deltify(rid, subsequent, 0); }else{ + search_doc_touch('e',rid,0); db_multi_exec( "REPLACE INTO event(type,mtime,objid,tagid,user,comment,bgcolor)" "VALUES('e',%.17g,%d,%d,%Q,%Q," " (SELECT value FROM tagxref WHERE tagid=%d AND rid=%d));", p->rEventDate, rid, tagid, p->zUser, p->zComment, Index: src/search.c ================================================================== --- src/search.c +++ src/search.c @@ -45,14 +45,15 @@ char *zPattern; /* The search pattern */ char *zMarkBegin; /* Start of a match */ char *zMarkEnd; /* End of a match */ char *zMarkGap; /* A gap between two matches */ unsigned fSrchFlg; /* Flags */ + int iScore; /* Score of the last match attempt */ + Blob snip; /* Snippet for the most recent match */ }; #define SRCHFLG_HTML 0x01 /* Escape snippet text for HTML */ -#define SRCHFLG_SCORE 0x02 /* Prepend the score to each snippet */ #define SRCHFLG_STATIC 0x04 /* The static gSearch object */ #endif /* @@ -92,10 +93,11 @@ if( p ){ fossil_free(p->zPattern); fossil_free(p->zMarkBegin); fossil_free(p->zMarkEnd); fossil_free(p->zMarkGap); + if( p->iScore ) blob_reset(&p->snip); memset(p, 0, sizeof(*p)); if( p!=&gSearch ) fossil_free(p); } } @@ -123,10 +125,11 @@ p->zPattern = z = mprintf("%s", zPattern); p->zMarkBegin = mprintf("%s", zMarkBegin); p->zMarkEnd = mprintf("%s", zMarkEnd); p->zMarkGap = mprintf("%s", zMarkGap); p->fSrchFlg = fSrchFlg; + blob_init(&p->snip, 0, 0); while( *z && p->nTerma[p->nTerm].z = z; for(i=1; ISALNUM(z[i]); i++){} @@ -156,24 +159,26 @@ } } /* ** Compare a search pattern against one or more input strings which -** collectively comprise a document. Return a match score. Optionally -** also return a "snippet". +** collectively comprise a document. Return a match score. Any +** postive value means there was a match. Zero means that one or +** more terms are missing. +** +** The score and a snippet are record for future use. ** ** Scoring: ** * All terms must match at least once or the score is zero ** * One point for each matching term ** * Extra points if consecutive words of the pattern are consecutive ** in the document */ -static int search_score( +static int search_match( Search *p, /* Search pattern and flags */ int nDoc, /* Number of strings in this document */ - const char **azDoc, /* Text of each string */ - Blob *pSnip /* If not NULL: Write a snippet here */ + const char **azDoc /* Text of each string */ ){ int score; /* Final score */ int i; /* Offset into current document */ int ii; /* Loop counter */ int j; /* Loop over search terms */ @@ -226,18 +231,17 @@ /* Finished search all documents. ** Every term must be seen or else the score is zero */ score = 1; for(j=0; jnTerm; j++) score *= anMatch[j]; - if( score==0 || pSnip==0 ) return score; + blob_reset(&p->snip); + p->iScore = score; + if( score==0 ) return score; /* Prepare a snippet that describes the matching text. */ - blob_init(pSnip, 0, 0); - if( p->fSrchFlg & SRCHFLG_SCORE ) blob_appendf(pSnip, "%08x", score); - while(1){ int iOfst; int iTail; int iBest; for(ii=0; iinTerm && anMatch[ii]==0; ii++){} @@ -272,11 +276,11 @@ if( iOfst<0 ) iOfst = 0; while( iOfst>0 && ISALNUM(zDoc[iOfst-1]) ) iOfst--; while( zDoc[iOfst] && !ISALNUM(zDoc[iOfst]) ) iOfst++; for(ii=0; ii0 || wantGap ) blob_append(pSnip, p->zMarkGap, -1); + if( iOfst>0 || wantGap ) blob_append(&p->snip, p->zMarkGap, -1); wantGap = zDoc[iTail]!=0; zDoc += iOfst; iTail -= iOfst; /* Add a snippet segment using characters iOfst..iOfst+iTail from zDoc */ @@ -285,53 +289,51 @@ for(j=0; jnTerm; j++){ int n = p->a[j].n; if( sqlite3_strnicmp(p->a[j].z, &zDoc[i], n)==0 && (!ISALNUM(zDoc[i+n]) || p->a[j].z[n]=='*') ){ - snippet_text_append(p, pSnip, zDoc, i); + snippet_text_append(p, &p->snip, zDoc, i); zDoc += i; iTail -= i; - blob_append(pSnip, p->zMarkBegin, -1); + blob_append(&p->snip, p->zMarkBegin, -1); if( p->a[j].z[n]=='*' ){ while( ISALNUM(zDoc[n]) ) n++; } - snippet_text_append(p, pSnip, zDoc, n); + snippet_text_append(p, &p->snip, zDoc, n); zDoc += n; iTail -= n; - blob_append(pSnip, p->zMarkEnd, -1); + blob_append(&p->snip, p->zMarkEnd, -1); i = -1; break; } /* end-if */ } /* end for(j) */ if( jnTerm ){ while( ISALNUM(zDoc[i]) && isnip, zDoc, iTail); } - if( wantGap ) blob_append(pSnip, p->zMarkGap, -1); + if( wantGap ) blob_append(&p->snip, p->zMarkGap, -1); return score; } /* -** COMMAND: test-snippet +** COMMAND: test-match ** -** Usage: fossil test-snippet SEARCHSTRING FILE1 FILE2 ... +** Usage: fossil test-match SEARCHSTRING FILE1 FILE2 ... */ -void test_snippet_cmd(void){ +void test_match_cmd(void){ Search *p; int i; Blob x; - Blob snip; int score; char *zDoc; int flg = 0; char *zBegin = (char*)find_option("begin",0,1); char *zEnd = (char*)find_option("end",0,1); char *zGap = (char*)find_option("gap",0,1); if( find_option("html",0,0)!=0 ) flg |= SRCHFLG_HTML; - if( find_option("score",0,0)!=0 ) flg |= SRCHFLG_SCORE; if( find_option("static",0,0)!=0 ) flg |= SRCHFLG_STATIC; verify_all_options(); if( g.argc<4 ) usage("SEARCHSTRING FILE1..."); if( zBegin==0 ) zBegin = "[["; if( zEnd==0 ) zEnd = "]]"; @@ -338,18 +340,18 @@ if( zGap==0 ) zGap = " ... "; p = search_init(g.argv[2], zBegin, zEnd, zGap, flg); for(i=3; iiScore); blob_reset(&x); if( score ){ - fossil_print("%.78c\n%s\n%.78c\n\n", '=', blob_str(&snip), '='); - blob_reset(&snip); + fossil_print("%.78c\n%s\n%.78c\n\n", '=', blob_str(&p->snip), '='); } } + search_end(p); } /* ** An SQL function to initialize the global search pattern: ** @@ -361,12 +363,12 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zPattern = 0; - const char *zBegin = ""; - const char *zEnd = ""; + const char *zBegin = ""; + const char *zEnd = ""; const char *zGap = " ... "; unsigned int flg = SRCHFLG_HTML; switch( argc ){ default: flg = (unsigned int)sqlite3_value_int(argv[4]); @@ -385,35 +387,45 @@ search_end(&gSearch); } } /* -** This is an SQLite function that scores its input using -** the pattern from the previous call to search_init(). +** Try to match the input text against the search parameters set up +** by the previous search_init() call. Remember the results globally. +** Return non-zero on a match and zero on a miss. +*/ +static void search_match_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSText = (const char*)sqlite3_value_text(argv[0]); + int rc; + if( zSText==0 ) return; + rc = search_match(&gSearch, 1, &zSText); + sqlite3_result_int(context, rc); +} + +/* +** These SQL functions return the results of the last +** call to the search_match() SQL function. */ static void search_score_sqlfunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - int isSnippet = sqlite3_user_data(context)!=0; - const char **azDoc; - int score; - int i; - Blob snip; - - if( gSearch.nTerm==0 ) return; - azDoc = fossil_malloc( sizeof(const char*)*(argc+1) ); - for(i=0; i0 ){ + sqlite3_result_text(context, blob_str(&gSearch.snip), -1, fossil_free); + blob_init(&gSearch.snip, 0, 0); } } /* ** This is an SQLite function that computes the searchable text. @@ -449,14 +461,18 @@ ** Register the "score()" SQL function to score its input text ** using the given Search object. Once this function is registered, ** do not delete the Search object. */ void search_sql_setup(sqlite3 *db){ - sqlite3_create_function(db, "score", -1, SQLITE_UTF8, 0, - search_score_sqlfunc, 0, 0); - sqlite3_create_function(db, "snippet", -1, SQLITE_UTF8, &gSearch, + static int once = 0; + if( once++ ) return; + sqlite3_create_function(db, "search_match", 1, SQLITE_UTF8, 0, + search_match_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0, search_score_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0, + search_snippet_sqlfunc, 0, 0); sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0, search_init_sqlfunc, 0, 0); sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0, search_stext_sqlfunc, 0, 0); sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0, @@ -551,28 +567,217 @@ /* ** Remove bits from srchFlags which are disallowed by either the ** current server configuration or by user permissions. */ unsigned int search_restrict(unsigned int srchFlags){ - if( (srchFlags & SRCH_CKIN)!=0 - && (g.perm.Read==0 || db_get_boolean("search-ci",0)==0) ){ + if( g.perm.Read==0 ) srchFlags &= ~(SRCH_CKIN|SRCH_DOC); + if( g.perm.RdTkt==0 ) srchFlags &= ~(SRCH_TKT); + if( g.perm.RdWiki==0 ) srchFlags &= ~(SRCH_WIKI); + if( search_index_exists() ) return srchFlags; + if( (srchFlags & SRCH_CKIN)!=0 && db_get_boolean("search-ci",0)==0 ){ srchFlags &= ~SRCH_CKIN; } - if( (srchFlags & SRCH_DOC)!=0 - && (g.perm.Read==0 || db_get_boolean("search-doc",0)==0) ){ + if( (srchFlags & SRCH_DOC)!=0 && db_get_boolean("search-doc",0)==0 ){ srchFlags &= ~SRCH_DOC; } - if( (srchFlags & SRCH_TKT)!=0 - && (g.perm.RdTkt==0 || db_get_boolean("search-tkt",0)==0) ){ + if( (srchFlags & SRCH_TKT)!=0 && db_get_boolean("search-tkt",0)==0 ){ srchFlags &= ~SRCH_TKT; } - if( (srchFlags & SRCH_WIKI)!=0 - && (g.perm.RdWiki==0 || db_get_boolean("search-wiki",0)==0) ){ + if( (srchFlags & SRCH_WIKI)!=0 && db_get_boolean("search-wiki",0)==0 ){ srchFlags &= ~SRCH_WIKI; } return srchFlags; } + +/* +** When this routine is called, there already exists a table +** +** x(label,url,score,date,snip). +** +** And the srchFlags parameter has been validated. This routine +** fills the X table with search results using a full-text scan. +** +** The companion indexed scan routine is search_indexed(). +*/ +static void search_fullscan( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags /* What to search over */ +){ + search_init(zPattern, "", "", " ... ", + SRCHFLG_STATIC|SRCHFLG_HTML); + if( (srchFlags & SRCH_DOC)!=0 ){ + char *zDocGlob = db_get("doc-glob",""); + char *zDocBr = db_get("doc-branch","trunk"); + if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){ + db_multi_exec( + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + ); + db_multi_exec( + "INSERT INTO x(label,url,score,date,snip)" + " SELECT printf('Document: %%s',foci.filename)," + " printf('%R/doc/%T/%%s',foci.filename)," + " search_score()," + " (SELECT datetime(event.mtime) FROM event" + " WHERE objid=symbolic_name_to_rid('trunk'))," + " search_snippet()" + " FROM foci CROSS JOIN blob" + " WHERE checkinID=symbolic_name_to_rid('trunk')" + " AND blob.uuid=foci.uuid" + " AND search_match(stext('d',blob.rid,foci.filename))" + " AND %z", + zDocBr, glob_expr("foci.filename", zDocGlob) + ); + } + } + if( (srchFlags & SRCH_WIKI)!=0 ){ + db_multi_exec( + "WITH wiki(name,rid,mtime) AS (" + " SELECT substr(tagname,6), tagxref.rid, max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " GROUP BY 1" + ")" + "INSERT INTO x(label,url,score,date,snip)" + " SELECT printf('Wiki: %%s',name)," + " printf('%R/wiki?name=%%s',urlencode(name))," + " search_score()," + " datetime(mtime)," + " search_snippet()" + " FROM wiki" + " WHERE search_match(stext('w',rid,name));" + ); + } + if( (srchFlags & SRCH_CKIN)!=0 ){ + db_multi_exec( + "WITH ckin(uuid,rid,mtime) AS (" + " SELECT blob.uuid, event.objid, event.mtime" + " FROM event, blob" + " WHERE event.type='ci'" + " AND blob.rid=event.objid" + ")" + "INSERT INTO x(label,url,score,date,snip)" + " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime))," + " printf('%R/timeline?c=%%s&n=8&y=ci',uuid)," + " search_score()," + " datetime(mtime)," + " search_snippet()" + " FROM ckin" + " WHERE search_match(stext('c',rid,NULL));" + ); + } + if( (srchFlags & SRCH_TKT)!=0 ){ + db_multi_exec( + "INSERT INTO x(label,url,score, date,snip)" + " SELECT printf('Ticket [%%.17s] on %%s'," + "tkt_uuid,datetime(tkt_mtime))," + " printf('%R/tktview/%%.20s',tkt_uuid)," + " search_score()," + " datetime(tkt_mtime)," + " search_snippet()" + " FROM ticket" + " WHERE search_match(stext('t',tkt_id,NULL));" + ); + } +} + +/* +** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')). +*/ +static void search_rank_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]); + int nVal = sqlite3_value_bytes(argv[0])/4; + int nTerm; /* Number of search terms in the query */ + int i; /* Loop counter */ + double r = 1.0; /* Score */ + + if( nVal<6 ) return; + if( aVal[1]!=1 ) return; + nTerm = aVal[0]; + r *= 1<<((30*(aVal[2]-1))/nTerm); + for(i=1; i<=nTerm; i++){ + int hits_this_row = aVal[3*i]; + int hits_all_rows = aVal[3*i+1]; + int rows_with_hit = aVal[3*i+2]; + double avg_hits_per_row = (double)hits_all_rows/(double)rows_with_hit; + r *= hits_this_row/avg_hits_per_row; + } +#define SEARCH_DEBUG_RANK 0 +#if SEARCH_DEBUG_RANK + { + Blob x; + blob_init(&x,0,0); + blob_appendf(&x,"%08x", (int)r); + for(i=0; i','')" + " FROM ftsidx, ftsdocs" + " WHERE ftsidx MATCH %Q" + " AND ftsdocs.rowid=ftsidx.docid", + zPattern + ); + if( srchFlags!=SRCH_ALL ){ + const char *zSep = " AND ("; + static const struct { unsigned m; char c; } aMask[] = { + { SRCH_CKIN, 'c' }, + { SRCH_DOC, 'd' }, + { SRCH_TKT, 't' }, + { SRCH_WIKI, 'w' }, + }; + int i; + for(i=0; i", "", " ... ", - SRCHFLG_STATIC|SRCHFLG_HTML|SRCHFLG_SCORE); - db_multi_exec( - "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" - "CREATE TEMP TABLE x(label TEXT,url TEXT,date TEXT,snip TEXT);" - ); - if( (srchFlags & SRCH_DOC)!=0 ){ - char *zDocGlob = db_get("doc-glob",""); - char *zDocBr = db_get("doc-branch","trunk"); - if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){ - db_multi_exec( - "INSERT INTO x(label,url,date,snip)" - " SELECT printf('Document: %%s',foci.filename)," - " printf('%R/doc/%T/%%s',foci.filename)," - " (SELECT datetime(event.mtime) FROM event" - " WHERE objid=symbolic_name_to_rid('trunk'))," - " snippet(stext('d',blob.rid,foci.filename))" - " FROM foci CROSS JOIN blob" - " WHERE checkinID=symbolic_name_to_rid('trunk')" - " AND blob.uuid=foci.uuid" - " AND %z", - zDocBr, glob_expr("foci.filename", zDocGlob) - ); - } - } - if( (srchFlags & SRCH_WIKI)!=0 ){ - db_multi_exec( - "WITH wiki(name,rid,mtime) AS (" - " SELECT substr(tagname,6), tagxref.rid, max(tagxref.mtime)" - " FROM tag, tagxref" - " WHERE tag.tagname GLOB 'wiki-*'" - " AND tagxref.tagid=tag.tagid" - " GROUP BY 1" - ")" - "INSERT INTO x(label,url,date,snip)" - " SELECT printf('Wiki: %%s',name)," - " printf('%R/wiki?name=%%s',urlencode(name))," - " datetime(mtime)," - " snippet(stext('w',rid,name))" - " FROM wiki;" - ); - } - if( (srchFlags & SRCH_CKIN)!=0 ){ - db_multi_exec( - "WITH ckin(uuid,rid,mtime) AS (" - " SELECT blob.uuid, event.objid, event.mtime" - " FROM event, blob" - " WHERE event.type='ci'" - " AND blob.rid=event.objid" - ")" - "INSERT INTO x(label,url,date,snip)" - " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime))," - " printf('%R/timeline?c=%%s&n=8&y=ci',uuid)," - " datetime(mtime)," - " snippet(stext('c',rid,NULL))" - " FROM ckin;" - ); - } - if( (srchFlags & SRCH_TKT)!=0 ){ - db_multi_exec( - "INSERT INTO x(label,url,date,snip)" - " SELECT printf('Ticket [%%.17s] on %%s'," - "tkt_uuid,datetime(tkt_mtime))," - " printf('%R/tktview/%%.20s',tkt_uuid)," - " datetime(tkt_mtime)," - " snippet(stext('t',tkt_id,NULL))" - " FROM ticket;" - ); - } - db_prepare(&q, "SELECT url, substr(snip,9), label" - " FROM x WHERE snip IS NOT NULL" - " ORDER BY substr(snip,1,8) DESC, date DESC;"); + db_multi_exec( + "CREATE TEMP TABLE x(label,url,score,date,snip);" + ); + if( !search_index_exists() ){ + search_fullscan(zPattern, srchFlags); + }else{ + search_update_index(srchFlags); + search_indexed(zPattern, srchFlags); + } + db_prepare(&q, "SELECT url, snip, label" + " FROM x" + " ORDER BY score DESC, date DESC;"); while( db_step(&q)==SQLITE_ROW ){ const char *zUrl = db_column_text(&q, 0); const char *zSnippet = db_column_text(&q, 1); const char *zLabel = db_column_text(&q, 2); if( nRow==0 ){ @
    } nRow++; - @

  1. %h(zLabel)
    %s(zSnippet)

    2. + @

  2. %h(zLabel)
    + @ %s(zSnippet)

    3. } db_finalize(&q); if( nRow ){ @
} @@ -789,18 +935,18 @@ ** zName Name of the object being searched. */ void search_stext( char cType, /* Type of document */ int rid, /* BLOB.RID or TAG.TAGID value for document */ - const char *zName, /* Name of the document */ + const char *zName, /* Auxiliary information */ Blob *pOut /* OUT: Initialize to the search text */ ){ blob_init(pOut, 0, 0); switch( cType ){ case 'd': { /* Documents */ Blob doc; - content_get(rid, &doc); + content_get(rid, &doc); blob_to_utf8_no_bom(&doc, 0); get_stext_by_mimetype(&doc, mimetype_from_name(zName), pOut); blob_reset(&doc); break; } @@ -860,63 +1006,355 @@ break; } } } -/* -** The arguments cType,rid,zName define an object that can be searched -** for. Return a URL (relative to the root of the Fossil project) that -** will jump to that document. -** -** Space to hold the returned string is obtained from mprintf() and should -** be freed by the caller using fossil_free() or the equivalent. -*/ -char *search_url( - char cType, /* Type of document */ - int rid, /* BLOB.RID or TAG.TAGID for the object */ - const char *zName /* Name of the object */ -){ - char *zUrl = 0; - switch( cType ){ - case 'd': { /* Documents */ - zUrl = db_text(0, - "SELECT printf('/doc/%%s%%s', substr(blob.uuid,20), %Q)" - " FROM mlink, blob" - " WHERE mlink.fid=%d AND mlink.mid=blob.rid", - zName, rid); - break; - } - case 'w': { /* Wiki */ - char *zId = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid); - zUrl = mprintf("/wiki?id=%z&name=%t", zId, zName); - break; - } - case 'c': { /* Ckeck-in Comment */ - char *zId = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid); - zUrl = mprintf("/info/%z", zId); - break; - } - case 't': { /* Tickets */ - char *zId = db_text(0, "SELECT tkt_uuid FROM ticket" - " WHERE tkt_id=%d", rid); - zUrl = mprintf("/tktview/%.20z", zId); - break; - } - } - return zUrl; -} - /* ** COMMAND: test-search-stext ** ** Usage: fossil test-search-stext TYPE ARG1 ARG2 */ void test_search_stext(void){ Blob out; - char *zUrl; db_find_and_open_repository(0,0); if( g.argc!=5 ) usage("TYPE RID NAME"); search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out); - zUrl = search_url(g.argv[2][0], atoi(g.argv[3]), g.argv[4]); - fossil_print("%s\n%z\n",blob_str(&out),zUrl); + fossil_print("%s\n",blob_str(&out)); blob_reset(&out); } + +/* The schema for the full-text index +*/ +static const char zFtsSchema[] = +@ -- One entry for each possible search result +@ CREATE TABLE IF NOT EXISTS "%w".ftsdocs( +@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid +@ type CHAR(1), -- Type of document +@ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document +@ name TEXT, -- Additional document description +@ idxed BOOLEAN, -- True if currently in the index +@ label TEXT, -- Label to print on search results +@ url TEXT, -- URL to access this document +@ mtime DATE, -- Date when document created +@ UNIQUE(type,rid) +@ ); +@ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0; +@ CREATE INDEX "%w".ftsdocName ON ftsdocs(name) WHERE type='w'; +@ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS +@ SELECT rowid, type, rid, name, idxed, label, url, mtime, +@ stext(type,rid,name) AS 'stext' +@ FROM ftsdocs; +@ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx +@ USING fts4(content="ftscontent", stext); +; +static const char zFtsDrop[] = +@ DROP TABLE IF EXISTS "%w".ftsidx; +@ DROP VIEW IF EXISTS "%w".ftscontent; +@ DROP TABLE IF EXISTS "%w".ftsdocs; +; + +/* +** Create or drop the tables associated with a full-text index. +*/ +void search_create_index(void){ + const char *zDb = db_name("repository"); + search_sql_setup(g.db); + db_multi_exec(zFtsSchema/*works-like:"%w%w%w%w%w"*/, + zDb, zDb, zDb, zDb, zDb); +} +void search_drop_index(void){ + const char *zDb = db_name("repository"); + db_multi_exec(zFtsDrop/*works-like:"%w%w%w"*/, zDb, zDb, zDb); +} + +/* +** Return true if the full-text search index exists +*/ +int search_index_exists(void){ + static int fExists = -1; + if( fExists<0 ) fExists = db_table_exists("repository","ftsdocs"); + return fExists; +} + +/* +** Fill the FTSDOCS table with unindexed entries for everything +** in the repository. This uses INSERT OR IGNORE so entries already +** in FTSDOCS are unchanged. +*/ +void search_fill_index(void){ + if( !search_index_exists() ) return; + search_sql_setup(g.db); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 'c', objid, 0 FROM event WHERE type='ci';" + ); + db_multi_exec( + "WITH latest_wiki(rid,name,mtime) AS (" + " SELECT tagxref.rid, substr(tag.tagname,6), max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " AND tagxref.value>0" + " GROUP BY 2" + ") INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed)" + " SELECT 'w', rid, name, 0 FROM latest_wiki;" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 't', tkt_id, 0 FROM ticket;" + ); +} + +/* +** The document described by cType,rid,zName is about to be added or +** updated. If the document has already been indexed, then unindex it +** now while we still have access to the old content. Add the document +** to the queue of documents that need to be indexed or reindexed. +*/ +void search_doc_touch(char cType, int rid, const char *zName){ + if( search_index_exists() ){ + char zType[2]; + zType[0] = cType; + zType[1] = 0; + db_multi_exec( + "DELETE FROM ftsidx WHERE docid IN" + " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)", + zType, rid + ); + db_multi_exec( + "REPLACE INTO ftsdocs(type,rid,name,idxed)" + " VALUES(%Q,%d,%Q,0)", + zType, rid, zName + ); + if( cType=='w' ){ + db_multi_exec( + "DELETE FROM ftsidx WHERE docid IN" + " (SELECT rowid FROM ftsdocs WHERE type='w' AND name=%Q AND idxed)", + zName + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='w' AND name=%Q AND rid!=%d", + zName, rid + ); + } + } +} + +/* +** If the doc-glob and doc-br settings are valid for document search +** and if the latest check-in on doc-br is in the unindexed set of +** check-ins, then update all 'd' entries in FTSDOCS that have +** changed. +*/ +static void search_update_doc_index(void){ + const char *zDocBr = db_get("doc-branch","trunk"); + int ckid = zDocBr ? symbolic_name_to_rid(zDocBr,"ci") : 0; + double rTime; + char *zBrUuid; + if( ckid==0 ) return; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE type='c' AND rid=%d" + " AND NOT idxed", ckid) ) return; + + /* If we get this far, it means that changes to 'd' entries are + ** required. */ + rTime = db_double(0.0, "SELECT mtime FROM event WHERE objid=%d", ckid); + zBrUuid = db_text("","SELECT substr(uuid,1,20) FROM blob WHERE rid=%d",ckid); + db_multi_exec( + "CREATE TEMP TABLE current_docs(rid INTEGER PRIMARY KEY, name);" + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + "INSERT OR IGNORE INTO current_docs(rid, name)" + " SELECT blob.rid, foci.filename FROM foci, blob" + " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid" + " AND %z", + ckid, glob_expr("foci.filename", db_get("doc-glob","")) + ); + db_multi_exec( + "DELETE FROM ftsidx WHERE docid IN" + " (SELECT rowid FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs))" + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs)" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,url,mtime)" + " SELECT 'd', rid, name, 0," + " printf('Document: %%s',name)," + " printf('/doc/%q/%%s',urlencode(name))," + " %.17g" + " FROM current_docs", + zBrUuid, rTime + ); + db_multi_exec( + "INSERT INTO ftsidx(docid,stext)" + " SELECT rowid, stext FROM ftscontent WHERE type='d' AND NOT idxed" + ); + db_multi_exec( + "UPDATE ftsdocs SET idxed=1 WHERE type='d' AND NOT idxed" + ); +} + +/* +** Deal with all of the unindexed 'c' terms in FTSDOCS +*/ +static void search_update_checkin_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(docid,stext)" + " SELECT rowid, stext('c',rid,NULL) FROM ftsdocs" + " WHERE type='c' AND NOT idxed;" + ); + db_multi_exec( + "REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)" + " SELECT ftsdocs.rowid, 1, 'c', ftsdocs.rid, NULL," + " printf('Check-in [%%.16s] on %%s',blob.uuid,datetime(event.mtime))," + " printf('/timeline?y=ci&n=9&c=%%.20s',blob.uuid)," + " event.mtime" + " FROM ftsdocs, event, blob" + " WHERE ftsdocs.type='c' AND NOT ftsdocs.idxed" + " AND event.objid=ftsdocs.rid" + " AND blob.rid=ftsdocs.rid" + ); +} + +/* +** Deal with all of the unindexed 't' terms in FTSDOCS +*/ +static void search_update_ticket_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(docid,stext)" + " SELECT rowid, stext('t',rid,NULL) FROM ftsdocs" + " WHERE type='t' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)" + " SELECT ftsdocs.rowid, 1, 't', ftsdocs.rid, NULL," + " printf('Ticket [%%.16s] on %%s',tkt_uuid,datetime(tkt_mtime))," + " printf('/tktview/%%.20s',tkt_uuid)," + " tkt_mtime" + " FROM ftsdocs, ticket" + " WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed" + " AND ticket.tkt_id=ftsdocs.rid" + ); +} + +/* +** Deal with all of the unindexed 'w' terms in FTSDOCS +*/ +static void search_update_wiki_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(docid,stext)" + " SELECT rowid, stext('w',rid,NULL) FROM ftsdocs" + " WHERE type='w' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)" + " SELECT ftsdocs.rowid, 1, 'w', ftsdocs.rid, ftsdocs.name," + " 'Wiki: '||ftsdocs.name," + " '/wiki?name='||urlencode(ftsdocs.name)," + " tagxref.mtime" + " FROM ftsdocs, tagxref" + " WHERE ftsdocs.type='w' AND NOT ftsdocs.idxed" + " AND tagxref.rid=ftsdocs.rid" + ); +} + +/* +** Deal with all of the unindexed entries in the FTSDOCS table - that +** is to say, all the entries with FTSDOCS.IDXED=0. Add them to the +** index. +*/ +void search_update_index(unsigned int srchFlags){ + if( !search_index_exists() ) return; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE NOT idxed") ) return; + search_sql_setup(g.db); + if( srchFlags & (SRCH_CKIN|SRCH_DOC) ){ + search_update_doc_index(); + search_update_checkin_index(); + } + if( srchFlags & SRCH_TKT ){ + search_update_ticket_index(); + } + if( srchFlags & SRCH_WIKI ){ + search_update_wiki_index(); + } +} + +/* +** COMMAND: test-fts +*/ +void test_fts_cmd(void){ + char *zSubCmd; + int i, n; + static const struct { int iCmd; const char *z; } aCmd[] = { + { 1, "create" }, + { 2, "drop" }, + { 3, "exists" }, + { 4, "fill" }, + { 8, "refill" }, + { 5, "pending" }, + { 7, "update" }, + }; + db_find_and_open_repository(0, 0); + if( g.argc<3 ) usage("SUBCMD ..."); + zSubCmd = g.argv[2]; + n = (int)strlen(zSubCmd); + for(i=0; i=ArraySize(aCmd) ){ + Blob all; + blob_init(&all,0,0); + for(i=0; i onoff_attribute("Search Wiki","search-wiki", "sw", 0, 0); @
@

+ @
+ if( P("fts0") ){ + search_drop_index(); + }else if( P("fts1") ){ + search_drop_index(); + search_create_index(); + search_fill_index(); + search_update_index(SRCH_ALL); + } + if( search_index_exists() ){ + @

Currently using an SQLite FTS4 search index. This makes search + @ run faster, especially on large repositories, but takes up space.

+ @

+ }else{ + @

The SQLite FTS4 search index is disabled. All searching will be + @ a full-text scan. This usually works fine, but can be slow for + @ larger repositories.

+ @

+ } @ style_footer(); } Index: src/style.c ================================================================== --- src/style.c +++ src/style.c @@ -1125,10 +1125,15 @@ }, { "th.sort.desc:after", "Descending sort column marker", @ content: '\2191'; }, + { "span.snippet>mark", + "Search markup", + @ background-color: inherit; + @ font-weight: bold; + }, { 0, 0, 0 } }; @@ -1149,23 +1154,47 @@ ); } } } +/* +** Search string zHaystack for zNeedle. zNeedle must be an isolated +** word with space or punctuation on either size. +** +** Return true if found. Return false if not found +*/ +static int containsString(const char *zHaystack, const char *zNeedle){ + char *z; + int n; + + while( zHaystack[0] ){ + z = strstr(zHaystack, zNeedle); + if( z==0 ) return 0; + n = (int)strlen(zNeedle); + if( (z==zHaystack || !fossil_isalnum(z[-1])) && !fossil_isalnum(z[n]) ){ + return 1; + } + zHaystack = z + n; + } + return 0; +} + + /* ** WEBPAGE: style.css */ void page_style_css(void){ Blob css; int i; cgi_set_content_type("text/css"); - blob_init(&css, db_get("css",(char*)builtin_text("skins/default/css.txt")), -1); + blob_init(&css,db_get("css",(char*)builtin_text("skins/default/css.txt")),-1); /* add special missing definitions */ for(i=1; cssDefaultList[i].elementClass; i++){ - if( strstr(blob_str(&css), cssDefaultList[i].elementClass)==0 ){ + char *z = blob_str(&css); + if( !containsString(z, cssDefaultList[i].elementClass) ){ blob_appendf(&css, "/* %s */\n%s {\n%s}\n", cssDefaultList[i].comment, cssDefaultList[i].elementClass, cssDefaultList[i].value); } Index: src/tkt.c ================================================================== --- src/tkt.c +++ src/tkt.c @@ -313,10 +313,11 @@ fossil_free(zTag); getAllTicketFields(); if( haveTicket==0 ) return; tktid = db_int(0, "SELECT tkt_id FROM ticket WHERE tkt_uuid=%Q", zTktUuid); + search_doc_touch('t', tktid, 0); if( haveTicketChng ){ db_multi_exec("DELETE FROM ticketchng WHERE tkt_id=%d;", tktid); } db_multi_exec("DELETE FROM ticket WHERE tkt_id=%d", tktid); tktid = 0; Index: win/Makefile.PellesCGMake ================================================================== --- win/Makefile.PellesCGMake +++ win/Makefile.PellesCGMake @@ -83,11 +83,11 @@ # define the SQLite files, which need special flags on compile SQLITESRC=sqlite3.c ORIGSQLITESRC=$(foreach sf,$(SQLITESRC),$(SRCDIR)$(sf)) SQLITEOBJ=$(foreach sf,$(SQLITESRC),$(sf:.c=.obj)) -SQLITEDEFINES=-DNDEBUG=1 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_ENABLE_LOCKING_STYLE=0 -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_OMIT_DEPRECATED -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_WIN32_NO_ANSI +SQLITEDEFINES=-DNDEBUG=1 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_ENABLE_LOCKING_STYLE=0 -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_OMIT_DEPRECATED -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_FTS4 -DSQLITE_WIN32_NO_ANSI # define the SQLite shell files, which need special flags on compile SQLITESHELLSRC=shell.c ORIGSQLITESHELLSRC=$(foreach sf,$(SQLITESHELLSRC),$(SRCDIR)$(sf)) SQLITESHELLOBJ=$(foreach sf,$(SQLITESHELLSRC),$(sf:.c=.obj)) Index: win/Makefile.dmc ================================================================== --- win/Makefile.dmc +++ win/Makefile.dmc @@ -24,11 +24,11 @@ CFLAGS = -o BCC = $(DMDIR)\bin\dmc $(CFLAGS) TCC = $(DMDIR)\bin\dmc $(CFLAGS) $(DMCDEF) $(SSL) $(INCL) LIBS = $(DMDIR)\extra\lib\ zlib wsock32 advapi32 -SQLITE_OPTIONS = -DNDEBUG=1 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_ENABLE_LOCKING_STYLE=0 -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_OMIT_DEPRECATED -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SQLITE_OPTIONS = -DNDEBUG=1 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_ENABLE_LOCKING_STYLE=0 -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_OMIT_DEPRECATED -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_FTS4 SHELL_OPTIONS = -Dmain=sqlite3_shell -DSQLITE_OMIT_LOAD_EXTENSION=1 -DUSE_SYSTEM_SQLITE=$(USE_SYSTEM_SQLITE) -DSQLITE_SHELL_DBNAME_PROC=fossil_open -Daccess=file_access -Dsystem=fossil_system -Dgetenv=fossil_getenv -Dfopen=fossil_fopen SRC = add_.c allrepo_.c attach_.c bag_.c bisect_.c blob_.c branch_.c browse_.c builtin_.c bundle_.c cache_.c captcha_.c cgi_.c checkin_.c checkout_.c clearsign_.c clone_.c comformat_.c configure_.c content_.c db_.c delta_.c deltacmd_.c descendants_.c diff_.c diffcmd_.c doc_.c encode_.c event_.c export_.c file_.c finfo_.c foci_.c fusefs_.c glob_.c graph_.c gzip_.c http_.c http_socket_.c http_ssl_.c http_transport_.c import_.c info_.c json_.c json_artifact_.c json_branch_.c json_config_.c json_diff_.c json_dir_.c json_finfo_.c json_login_.c json_query_.c json_report_.c json_status_.c json_tag_.c json_timeline_.c json_user_.c json_wiki_.c leaf_.c loadctrl_.c login_.c lookslike_.c main_.c manifest_.c markdown_.c markdown_html_.c md5_.c merge_.c merge3_.c moderate_.c name_.c path_.c pivot_.c popen_.c pqueue_.c printf_.c publish_.c purge_.c rebuild_.c regexp_.c report_.c rss_.c schema_.c search_.c setup_.c sha1_.c shun_.c sitemap_.c skins_.c sqlcmd_.c stash_.c stat_.c statrep_.c style_.c sync_.c tag_.c tar_.c th_main_.c timeline_.c tkt_.c tktsetup_.c undo_.c unicode_.c update_.c url_.c user_.c utf8_.c util_.c verify_.c vfile_.c wiki_.c wikiformat_.c winfile_.c winhttp_.c wysiwyg_.c xfer_.c xfersetup_.c zip_.c Index: win/Makefile.mingw ================================================================== --- win/Makefile.mingw +++ win/Makefile.mingw @@ -2024,10 +2024,11 @@ -DSQLITE_ENABLE_LOCKING_STYLE=0 \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_DEFAULT_FILE_FORMAT=4 \ -DSQLITE_OMIT_DEPRECATED \ -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS4 \ -DSQLITE_WIN32_NO_ANSI \ -D_HAVE__MINGW_H \ -DSQLITE_USE_MALLOC_H \ -DSQLITE_USE_MSIZE Index: win/Makefile.msc ================================================================== --- win/Makefile.msc +++ win/Makefile.msc @@ -189,10 +189,11 @@ /DSQLITE_ENABLE_LOCKING_STYLE=0 \ /DSQLITE_THREADSAFE=0 \ /DSQLITE_DEFAULT_FILE_FORMAT=4 \ /DSQLITE_OMIT_DEPRECATED \ /DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + /DSQLITE_ENABLE_FTS4 \ /DSQLITE_WIN32_NO_ANSI SHELL_OPTIONS = /Dmain=sqlite3_shell \ /DSQLITE_OMIT_LOAD_EXTENSION=1 \ /DUSE_SYSTEM_SQLITE=$(USE_SYSTEM_SQLITE) \