Fossil

Check-in [15a53308]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The emailblob table holds content a little while before deleting it. Change the /test-emailblob page to /emailblob, though it is still only accessible by the administrator.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 15a533086928eb88a3448390e752e0588346cde2caaee8ae39abb08eda07a567
User & Date: drh 2018-07-17 19:45:30.646
Context
2018-07-17
20:59
Store the uncompressed artifact size as an integer field in the emailblob table. ... (check-in: 34b95026 user: drh tags: trunk)
19:45
The emailblob table holds content a little while before deleting it. Change the /test-emailblob page to /emailblob, though it is still only accessible by the administrator. ... (check-in: 15a53308 user: drh tags: trunk)
18:32
Improved tracing capabilities for email notifications. Add the SelectAll button to the webmail display. ... (check-in: f880aa82 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/smtp.c.
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
/*
** Schema used by the email processing system.
*/
static const char zEmailSchema[] = 
@ -- bulk storage is in a separate table.  This table can store either
@ -- the body of email messages or transcripts of smtp sessions.
@ CREATE TABLE IF NOT EXISTS repository.emailblob(
@   emailid INTEGER PRIMARY KEY,  -- numeric idea for the entry
@   enref INT,                    -- Number of references to this blob
@   ets INT,                      -- Corresponding transcript, or NULL
@   etime INT,                    -- insertion time, secs since 1970
@   etxt TEXT                     -- content of this entry
@ );
@
@ -- One row for each mailbox entry.  All users emails are stored in







|







635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
/*
** Schema used by the email processing system.
*/
static const char zEmailSchema[] = 
@ -- bulk storage is in a separate table.  This table can store either
@ -- the body of email messages or transcripts of smtp sessions.
@ CREATE TABLE IF NOT EXISTS repository.emailblob(
@   emailid INTEGER PRIMARY KEY AUTOINCREMENT,  -- numeric idea for the entry
@   enref INT,                    -- Number of references to this blob
@   ets INT,                      -- Corresponding transcript, or NULL
@   etime INT,                    -- insertion time, secs since 1970
@   etxt TEXT                     -- content of this entry
@ );
@
@ -- One row for each mailbox entry.  All users emails are stored in
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697




698
699
700
701
702
703
704
@ );
@
@ -- Triggers to automatically keep the emailblob.enref field up to date
@ -- as entries in the emailblob, emailbox, and emailoutq tables are
@ -- deleted.
@ CREATE TRIGGER IF NOT EXISTS repository.emailblob_d1
@ AFTER DELETE ON emailblob BEGIN
@   DELETE FROM emailblob WHERE enref<=1 AND emailid=old.ets;
@   UPDATE emailblob SET enref=enref-1 WHERE emailid=old.ets;
@ END;
@ CREATE TRIGGER IF NOT EXISTS repository.emailbox_d1
@ AFTER DELETE ON emailbox BEGIN
@   DELETE FROM emailblob WHERE enref<=1 AND emailid=old.emsgid;
@   UPDATE emailblob SET enref=enref-1 WHERE emailid=old.emsgid;
@ END;
@ CREATE TRIGGER IF NOT EXISTS repository.emailoutq_d1
@ AFTER DELETE ON emailoutq BEGIN
@   DELETE FROM emailblob WHERE enref<=1 AND emailid IN (old.ets,old.emsgid);
@   UPDATE emailblob SET enref=enref-1 WHERE emailid IN (old.ets,old.emsgid);
@ END;




;

/*
** Code used to delete the email tables.
*/
static const char zEmailDrop[] =
@ DROP TABLE IF EXISTS emailblob;







<




<




<


>
>
>
>







678
679
680
681
682
683
684

685
686
687
688

689
690
691
692

693
694
695
696
697
698
699
700
701
702
703
704
705
@ );
@
@ -- Triggers to automatically keep the emailblob.enref field up to date
@ -- as entries in the emailblob, emailbox, and emailoutq tables are
@ -- deleted.
@ CREATE TRIGGER IF NOT EXISTS repository.emailblob_d1
@ AFTER DELETE ON emailblob BEGIN

@   UPDATE emailblob SET enref=enref-1 WHERE emailid=old.ets;
@ END;
@ CREATE TRIGGER IF NOT EXISTS repository.emailbox_d1
@ AFTER DELETE ON emailbox BEGIN

@   UPDATE emailblob SET enref=enref-1 WHERE emailid=old.emsgid;
@ END;
@ CREATE TRIGGER IF NOT EXISTS repository.emailoutq_d1
@ AFTER DELETE ON emailoutq BEGIN

@   UPDATE emailblob SET enref=enref-1 WHERE emailid IN (old.ets,old.emsgid);
@ END;
@
@ -- An index on the emailblob entries which are unreferenced.
@ CREATE INDEX IF NOT EXISTS repository.emailblob_nref ON emailblob(enref)
@ WHERE enref<=0;
;

/*
** Code used to delete the email tables.
*/
static const char zEmailDrop[] =
@ DROP TABLE IF EXISTS emailblob;
1056
1057
1058
1059
1060
1061
1062

1063
1064
1065
1066
1067
1068
1069
/*
** The SmtpServer object contains a complete incoming email.
** Add this email to the database.
*/
static void smtp_server_route_incoming(SmtpServer *p, int bFinish){
  Stmt s;
  int i;

  if( p->zFrom
   && p->nTo
   && blob_size(&p->msg)
   && (p->srvrFlags & SMTPSRV_DRYRUN)==0
  ){
    db_begin_write();
    if( p->idTranscript==0 ) smtp_server_schema(0);







>







1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
/*
** The SmtpServer object contains a complete incoming email.
** Add this email to the database.
*/
static void smtp_server_route_incoming(SmtpServer *p, int bFinish){
  Stmt s;
  int i;
  int nEtsStart = p->nEts;
  if( p->zFrom
   && p->nTo
   && blob_size(&p->msg)
   && (p->srvrFlags & SMTPSRV_DRYRUN)==0
  ){
    db_begin_write();
    if( p->idTranscript==0 ) smtp_server_schema(0);
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
        db_step(&s);
        db_reset(&s);
        p->idTranscript = db_last_insert_rowid();
        db_multi_exec(
          "UPDATE emailblob SET enref=%d WHERE emailid=%lld",
          p->nEts, p->idTranscript);
      }
      smtp_server_send(p, "221-Transcript id %lld nref %d\r\n",
         p->idTranscript, p->nEts);
    }
    db_bind_int64(&s, ":ets", p->idTranscript);
    db_bind_str(&s, ":etxt", &p->msg);
    db_step(&s);
    db_finalize(&s);
    p->idMsg = db_last_insert_rowid();








|
|







1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
        db_step(&s);
        db_reset(&s);
        p->idTranscript = db_last_insert_rowid();
        db_multi_exec(
          "UPDATE emailblob SET enref=%d WHERE emailid=%lld",
          p->nEts, p->idTranscript);
      }
      /* smtp_server_send(p, "221-Transcript id %lld nref %d\r\n",
      **   p->idTranscript, p->nEts); */
    }
    db_bind_int64(&s, ":ets", p->idTranscript);
    db_bind_str(&s, ":etxt", &p->msg);
    db_step(&s);
    db_finalize(&s);
    p->idMsg = db_last_insert_rowid();

1117
1118
1119
1120
1121
1122
1123






1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140

1141
1142
1143
1144

1145
1146
1147
1148
1149
1150

1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170






1171
1172
1173
1174
1175
1176
1177
        "UPDATE emailblob SET enref=%d WHERE emailid=%lld", 
        p->nRef, p->idMsg
      );
    }else{
      db_multi_exec(
        "DELETE FROM emailblob WHERE emailid=%lld", p->idMsg
      );






    }

    /* Finish the transaction after all changes are implemented */
    db_commit_transaction();
  }
  smtp_server_clear(p, SMTPSRV_CLEAR_MSG);
}

/*
** COMMAND: test-emailblob-refcheck
**
** Usage: %fossil test-emailblob-refcheck [--repair] [--full]
**
** Verify that the emailblob.enref field is correct.  Report any errors.
** Use the --repair command to fix up the enref field.  The --full option
** gives a full report showing the enref value on all entries in the
** emailblob table.

*/
void test_refcheck_emailblob(void){
  int doRepair;
  int fullReport;

  Blob sql;
  Stmt q;
  int nErr = 0;
  db_find_and_open_repository(0, 0);
  fullReport = find_option("full",0,0)!=0;
  doRepair = find_option("repair",0,0)!=0;

  verify_all_options();
  if( !db_table_exists("repository","emailblob") ){
    fossil_print("emailblob table is not configured - nothing to check\n");
    return;
  }
  db_multi_exec(
    "CREATE TEMP TABLE refcnt(id INTEGER PRIMARY KEY, n);"
    "INSERT INTO refcnt SELECT ets, count(*) FROM ("
    "  SELECT ets FROM emailblob"
    "  UNION ALL"
    "  SELECT emsgid FROM emailbox"
    "  UNION ALL"
    "  SELECT emsgid FROM emailoutq"
    ") WHERE ets IS NOT NULL GROUP BY 1;"
    "INSERT OR IGNORE INTO refcnt(id,n) SELECT emailid, 0 FROM emailblob;"
  );
  if( doRepair ){
    db_multi_exec(
      "UPDATE emailblob SET enref=(SELECT n FROM refcnt WHERE id=emailid)"
    );






  }
  blob_init(&sql, 0, 0);
  blob_append_sql(&sql, 
    "SELECT a.emailid, a.enref, b.n"
    "  FROM emailblob AS a JOIN refcnt AS b ON a.emailid=b.id"
  );
  if( !fullReport ){







>
>
>
>
>
>











|




|
>




>






>




















>
>
>
>
>
>







1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
        "UPDATE emailblob SET enref=%d WHERE emailid=%lld", 
        p->nRef, p->idMsg
      );
    }else{
      db_multi_exec(
        "DELETE FROM emailblob WHERE emailid=%lld", p->idMsg
      );
      p->nEts = nEtsStart;
    }

    /* Clean out legacy entries */
    if( bFinish ){
      db_multi_exec("DELETE FROM emailblob WHERE enref<=0");
    }

    /* Finish the transaction after all changes are implemented */
    db_commit_transaction();
  }
  smtp_server_clear(p, SMTPSRV_CLEAR_MSG);
}

/*
** COMMAND: test-emailblob-refcheck
**
** Usage: %fossil test-emailblob-refcheck [--repair] [--full] [--clean]
**
** Verify that the emailblob.enref field is correct.  Report any errors.
** Use the --repair command to fix up the enref field.  The --full option
** gives a full report showing the enref value on all entries in the
** emailblob table.  If the --clean flags is used together with --repair,
** then emailblob table entires with enref==0 are removed.
*/
void test_refcheck_emailblob(void){
  int doRepair;
  int fullReport;
  int doClean;
  Blob sql;
  Stmt q;
  int nErr = 0;
  db_find_and_open_repository(0, 0);
  fullReport = find_option("full",0,0)!=0;
  doRepair = find_option("repair",0,0)!=0;
  doClean = find_option("clean",0,0)!=0;
  verify_all_options();
  if( !db_table_exists("repository","emailblob") ){
    fossil_print("emailblob table is not configured - nothing to check\n");
    return;
  }
  db_multi_exec(
    "CREATE TEMP TABLE refcnt(id INTEGER PRIMARY KEY, n);"
    "INSERT INTO refcnt SELECT ets, count(*) FROM ("
    "  SELECT ets FROM emailblob"
    "  UNION ALL"
    "  SELECT emsgid FROM emailbox"
    "  UNION ALL"
    "  SELECT emsgid FROM emailoutq"
    ") WHERE ets IS NOT NULL GROUP BY 1;"
    "INSERT OR IGNORE INTO refcnt(id,n) SELECT emailid, 0 FROM emailblob;"
  );
  if( doRepair ){
    db_multi_exec(
      "UPDATE emailblob SET enref=(SELECT n FROM refcnt WHERE id=emailid)"
    );
    if( doClean ){
      db_multi_exec(
        "UPDATE emailblob SET ets=NULL WHERE enref<=0;"
        "DELETE FROM emailblob WHERE enref<=0;"
      );
    }
  }
  blob_init(&sql, 0, 0);
  blob_append_sql(&sql, 
    "SELECT a.emailid, a.enref, b.n"
    "  FROM emailblob AS a JOIN refcnt AS b ON a.emailid=b.id"
  );
  if( !fullReport ){
Changes to src/webmail.c.
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
  @   }
  @ }
  @ </script>
  db_end_transaction(0);
}

/*
** WEBPAGE:  test-emailblob
**
** This page, accessible only to administrators, allows easy viewing of
** the emailblob table - the table that contains the text of email messages
** both inbound and outbound, and transcripts of SMTP sessions.
**
**    id=N          Show the text of emailblob with emailid==N
**    
*/
void webmail_emailblob_page(void){
  int id = atoi(PD("id","0"));
  Stmt q;
  login_check_credentials();
  if( !g.perm.Setup ){
    login_needed(0);
    return;
  }
  add_content_sql_commands(g.db);
  style_header("emailblob table");
  if( id>0 ){
    style_submenu_element("Index", "%R/test-emailblob");
    @ <ul>
    db_prepare(&q, "SELECT emailid FROM emailblob WHERE ets=%d", id);
    while( db_step(&q)==SQLITE_ROW ){
      int id = db_column_int(&q, 0);
      @ <li> <a href="%R/test-emailblob?id=%d(id)">emailblob entry %d(id)</a>
    }
    db_finalize(&q);
    db_prepare(&q, "SELECT euser, estate FROM emailbox WHERE emsgid=%d", id);
    while( db_step(&q)==SQLITE_ROW ){
      const char *zUser = db_column_text(&q, 0);
      int e = db_column_int(&q, 1);
      @ <li> emailbox for %h(zUser) state %d(e)







|



















|




|







757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
  @   }
  @ }
  @ </script>
  db_end_transaction(0);
}

/*
** WEBPAGE:  emailblob
**
** This page, accessible only to administrators, allows easy viewing of
** the emailblob table - the table that contains the text of email messages
** both inbound and outbound, and transcripts of SMTP sessions.
**
**    id=N          Show the text of emailblob with emailid==N
**    
*/
void webmail_emailblob_page(void){
  int id = atoi(PD("id","0"));
  Stmt q;
  login_check_credentials();
  if( !g.perm.Setup ){
    login_needed(0);
    return;
  }
  add_content_sql_commands(g.db);
  style_header("emailblob table");
  if( id>0 ){
    style_submenu_element("Index", "%R/emailblob");
    @ <ul>
    db_prepare(&q, "SELECT emailid FROM emailblob WHERE ets=%d", id);
    while( db_step(&q)==SQLITE_ROW ){
      int id = db_column_int(&q, 0);
      @ <li> <a href="%R/emailblob?id=%d(id)">emailblob entry %d(id)</a>
    }
    db_finalize(&q);
    db_prepare(&q, "SELECT euser, estate FROM emailbox WHERE emsgid=%d", id);
    while( db_step(&q)==SQLITE_ROW ){
      const char *zUser = db_column_text(&q, 0);
      int e = db_column_int(&q, 1);
      @ <li> emailbox for %h(zUser) state %d(e)
817
818
819
820
821
822
823
824

825
826
827
828
829
830
831
832

833
834
835

836



837

838
839
840
841
842
843
844
    while( db_step(&q)==SQLITE_ROW ){
      const char *zContent = db_column_text(&q, 0);
      @ <pre>%h(zContent)</pre>
    }
    db_finalize(&q);
  }else{
    db_prepare(&q,
       "SELECT emailid, enref, ets, datetime(etime,'unixepoch')"

       " FROM emailblob ORDER BY etime DESC, emailid DESC");
    @ <table border="1" cellpadding="5" cellspacing="0">
    @ <tr><th> emailid <th> enref <th> ets <th> etime</tr>
    while( db_step(&q)==SQLITE_ROW ){
      int id = db_column_int(&q, 0);
      int nref = db_column_int(&q, 1);
      int ets = db_column_int(&q, 2);
      const char *zDate = db_column_text(&q, 3);

      @ <tr>
      @  <td><a href="%R/test-emailblob?id=%d(id)">%d(id)</a>
      @  <td>%d(nref)</td>

      @  <td>%d(ets)</td>



      @  <td>%h(zDate)</td>

      @ </tr>
    }
    @ </table>
    db_finalize(&q);
  }
  style_footer();
}







|
>


|





>

|
|
>
|
>
>
>

>







817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
    while( db_step(&q)==SQLITE_ROW ){
      const char *zContent = db_column_text(&q, 0);
      @ <pre>%h(zContent)</pre>
    }
    db_finalize(&q);
  }else{
    db_prepare(&q,
       "SELECT emailid, enref, ets, datetime(etime,'unixepoch'),"
       " length(etxt)"
       " FROM emailblob ORDER BY etime DESC, emailid DESC");
    @ <table border="1" cellpadding="5" cellspacing="0">
    @ <tr><th> emailid <th> enref <th> ets <th> etime <th> size </tr>
    while( db_step(&q)==SQLITE_ROW ){
      int id = db_column_int(&q, 0);
      int nref = db_column_int(&q, 1);
      int ets = db_column_int(&q, 2);
      const char *zDate = db_column_text(&q, 3);
      int sz = db_column_int(&q,4);
      @ <tr>
      @  <td align="right"><a href="%R/emailblob?id=%d(id)">%d(id)</a>
      @  <td align="right">%d(nref)</td>
      if( ets>0 ){
        @  <td align="right">%d(ets)</td>
      }else{
        @  <td>&nbsp;</td>
      }
      @  <td>%h(zDate)</td>
      @  <td align="right">%,d(sz)</td>
      @ </tr>
    }
    @ </table>
    db_finalize(&q);
  }
  style_footer();
}