ccnr_store.c

Go to the documentation of this file.
00001 /**
00002  * @file ccnr_store.c
00003  * 
00004  * Part of ccnr -  CCNx Repository Daemon.
00005  *
00006  */
00007 
00008 /*
00009  * Copyright (C) 2011 Palo Alto Research Center, Inc.
00010  *
00011  * This work is free software; you can redistribute it and/or modify it under
00012  * the terms of the GNU General Public License version 2 as published by the
00013  * Free Software Foundation.
00014  * This work is distributed in the hope that it will be useful, but WITHOUT ANY
00015  * WARRANTY; without even the implied warranty of MERCHANTABILITY or
00016  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
00017  * for more details. You should have received a copy of the GNU General Public
00018  * License along with this program; if not, write to the
00019  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
00020  * Boston, MA 02110-1301, USA.
00021  */
00022  
00023 #include <errno.h>
00024 #include <fcntl.h>
00025 #include <limits.h>
00026 #include <netdb.h>
00027 #include <poll.h>
00028 #include <signal.h>
00029 #include <stddef.h>
00030 #include <stdint.h>
00031 #include <stdio.h>
00032 #include <stdlib.h>
00033 #include <string.h>
00034 #include <time.h>
00035 #include <unistd.h>
00036 #include <arpa/inet.h>
00037 #include <sys/time.h>
00038 #include <sys/socket.h>
00039 #include <sys/stat.h>
00040 #include <sys/types.h>
00041 #include <sys/un.h>
00042 #include <netinet/in.h>
00043 
00044 #include <ccn/bloom.h>
00045 #include <ccn/btree_content.h>
00046 #include <ccn/ccn.h>
00047 #include <ccn/ccn_private.h>
00048 #include <ccn/charbuf.h>
00049 #include <ccn/face_mgmt.h>
00050 #include <ccn/hashtb.h>
00051 #include <ccn/indexbuf.h>
00052 #include <ccn/schedule.h>
00053 #include <ccn/reg_mgmt.h>
00054 #include <ccn/uri.h>
00055 
00056 #include "ccnr_private.h"
00057 
00058 #include "ccnr_stats.h"
00059 #include "ccnr_store.h"
00060 #include "ccnr_init.h"
00061 #include "ccnr_link.h"
00062 #include "ccnr_util.h"
00063 #include "ccnr_proto.h"
00064 #include "ccnr_msg.h"
00065 #include "ccnr_sync.h"
00066 #include "ccnr_match.h"
00067 #include "ccnr_sendq.h"
00068 #include "ccnr_io.h"
00069 
00070 struct content_entry {
00071     ccnr_accession accession;   /**< permanent repository id */
00072     ccnr_cookie cookie;         /**< for in-memory references */
00073     int flags;                  /**< see below - use accessor functions */
00074     int size;                   /**< size of ContentObject */
00075     struct ccn_charbuf *flatname; /**< for skiplist, et. al. */
00076     struct ccn_charbuf *cob;    /**< may contain ContentObject, or be NULL */
00077 };
00078 
00079 static const unsigned char *bogon = NULL;
00080 
00081 static int
00082 r_store_set_flatname(struct ccnr_handle *h, struct content_entry *content,
00083                      struct ccn_parsed_ContentObject *pco);
00084 static int
00085 r_store_content_btree_insert(struct ccnr_handle *h,
00086                              struct content_entry *content,
00087                              struct ccn_parsed_ContentObject *pco,
00088                              ccnr_accession *accession);
00089 
00090 #define FAILIF(cond) do {} while ((cond) && r_store_fatal(h, __func__, __LINE__))
00091 #define CHKSYS(res) FAILIF((res) == -1)
00092 #define CHKRES(res) FAILIF((res) < 0)
00093 #define CHKPTR(p)   FAILIF((p) == NULL)
00094 
00095 static int
00096 r_store_fatal(struct ccnr_handle *h, const char *fn, int lineno)
00097 {
00098     if (h != NULL) {
00099         ccnr_msg(h,
00100                  "fatal error in %s, line %d, errno %d%s",
00101                  fn, lineno, errno, strerror(errno));
00102     }
00103     abort();
00104     return(0);
00105 }
00106 
00107 PUBLIC ccnr_accession
00108 r_store_content_accession(struct ccnr_handle *h, struct content_entry *content)
00109 {
00110     return(content->accession);
00111 }
00112 
00113 PUBLIC ccnr_cookie
00114 r_store_content_cookie(struct ccnr_handle *h, struct content_entry *content)
00115 {
00116     return(content->cookie);
00117 }
00118 
00119 PUBLIC size_t
00120 r_store_content_size(struct ccnr_handle *h, struct content_entry *content)
00121 {
00122     return(content->size);
00123 }
00124 
00125 static off_t
00126 r_store_offset_from_accession(struct ccnr_handle *h, ccnr_accession a)
00127 {
00128     return(a & ((((ccnr_accession)1) << 48) - 1));
00129 }
00130 
00131 static unsigned
00132 r_store_repofile_from_accession(struct ccnr_handle *h, ccnr_accession a)
00133 {
00134     /* Initially this should always be 1 */
00135     return(a >> 48);
00136 }
00137 
00138 
00139 static const unsigned char *
00140 r_store_content_mapped(struct ccnr_handle *h, struct content_entry *content)
00141 {
00142     return(NULL);
00143 }
00144 
00145 static const unsigned char *
00146 r_store_content_read(struct ccnr_handle *h, struct content_entry *content)
00147 {
00148     unsigned repofile;
00149     off_t offset;
00150     struct ccn_charbuf *cob = NULL;
00151     ssize_t rres = 0;
00152     int fd = -1;
00153     unsigned char buf[8800];
00154     struct ccn_skeleton_decoder decoder = {0};
00155     struct ccn_skeleton_decoder *d = &decoder;
00156     ssize_t dres;
00157     
00158     repofile = r_store_repofile_from_accession(h, content->accession);
00159     offset = r_store_offset_from_accession(h, content->accession);
00160     if (repofile != 1)
00161         goto Bail;
00162     if (content->cob != NULL)
00163         goto Bail;
00164     fd = r_io_repo_data_file_fd(h, repofile, 0);
00165     if (fd == -1)
00166         goto Bail;
00167     cob = ccn_charbuf_create();
00168     if (cob == NULL)
00169         goto Bail;
00170     if (content->size > 0) {
00171         if (ccn_charbuf_reserve(cob, content->size) == NULL)
00172             goto Bail;
00173         rres = pread(fd, cob->buf, content->size, offset);
00174         if (rres == content->size) {
00175             cob->length = content->size;
00176             content->cob = cob;
00177             h->cob_count++;
00178             return(cob->buf);
00179         }
00180         if (rres == -1)
00181             ccnr_msg(h, "r_store_content_read %u :%s (errno = %d)",
00182                      fd, strerror(errno), errno);
00183         else
00184             ccnr_msg(h, "r_store_content_read %u expected %d bytes, but got %d",
00185                      fd, (int)content->size, (int)rres);
00186     } else {
00187         rres = pread(fd, buf, 8800, offset); // XXX - should be symbolic
00188         if (rres == -1) {
00189             ccnr_msg(h, "r_store_content_read %u :%s (errno = %d)",
00190                      fd, strerror(errno), errno);
00191             goto Bail;
00192         }
00193         dres = ccn_skeleton_decode(d, buf, rres);
00194         if (d->state != 0) {
00195             ccnr_msg(h, "r_store_content_read %u : error parsing cob", fd);
00196             goto Bail;
00197         }
00198         content->size = dres;
00199         if (ccn_charbuf_append(cob, buf, dres) < 0)
00200             goto Bail;
00201         content->cob = cob;
00202         h->cob_count++;
00203         return(cob->buf);        
00204     }
00205 Bail:
00206     ccn_charbuf_destroy(&cob);
00207     return(NULL);
00208 }
00209 
00210 /**
00211  *  If the content appears to be safely stored in the repository,
00212  *  removes any buffered copy.
00213  * @returns 0 if buffer was removed, -1 if not.
00214  */
00215 PUBLIC int
00216 r_store_content_trim(struct ccnr_handle *h, struct content_entry *content)
00217 {
00218     if (content->accession != CCNR_NULL_ACCESSION && content->cob != NULL) {
00219         ccn_charbuf_destroy(&content->cob);
00220         h->cob_count--;
00221         return(0);
00222     }
00223     return(-1);
00224 }
00225 
00226 /**
00227  *  Evict recoverable content from in-memory buffers
00228  */
00229 PUBLIC void
00230 r_store_trim(struct ccnr_handle *h, unsigned long limit)
00231 {
00232     struct content_entry *content = NULL;
00233     int checklimit;
00234     unsigned before;
00235     unsigned rover;
00236     unsigned mask;
00237     
00238     r_store_index_needs_cleaning(h);
00239     before = h->cob_count;
00240     if (before <= limit)
00241         return;
00242     checklimit = h->cookie_limit;
00243     mask = h->cookie_limit - 1;
00244     for (rover = (h->trim_rover & mask);
00245          checklimit > 0 && h->cob_count > limit;
00246          checklimit--, rover = (rover + 1) & mask) {
00247         content = h->content_by_cookie[rover];
00248         if (content != NULL)
00249             r_store_content_trim(h, content);
00250     }
00251     h->trim_rover = rover;
00252     if (CCNSHOULDLOG(h, sdf, CCNL_FINER))
00253         ccnr_msg(h, "trimmed %u cobs", before - h->cob_count);
00254 }
00255 
00256 /**
00257  *  Get the base address of the content object
00258  *
00259  * This may involve reading the object in.  Caller should not assume that
00260  * the address will stay valid after it relinquishes control, either by
00261  * returning or by calling routines that might invalidate objects.
00262  *
00263  */
00264 PUBLIC const unsigned char *
00265 r_store_content_base(struct ccnr_handle *h, struct content_entry *content)
00266 {
00267     const unsigned char *ans = NULL;
00268     
00269     if (content->cob != NULL && content->cob->length == content->size) {
00270         ans = content->cob->buf;
00271         goto Finish;
00272     }
00273     if (content->accession == CCNR_NULL_ACCESSION)
00274         goto Finish;
00275     ans = r_store_content_mapped(h, content);
00276     if (ans != NULL)
00277         goto Finish;
00278     ans = r_store_content_read(h, content);
00279 Finish:
00280     if (ans != NULL) {
00281         /* Sanity check - make sure first 2 and last 2 bytes are good */
00282         if (content->size < 5 || ans[0] != 0x04 || ans[1] != 0x82 ||
00283             ans[content->size - 1] != 0 || ans[content->size - 2] != 0) {
00284             bogon = ans; /* for debugger */
00285             ans = NULL;
00286         }
00287     }
00288     if (ans == NULL || CCNSHOULDLOG(h, xxxx, CCNL_FINEST))
00289         ccnr_msg(h, "r_store_content_base.%d returning %p (acc=0x%jx, cookie=%u)",
00290                  __LINE__,
00291                  ans,
00292                  ccnr_accession_encode(h, content->accession),
00293                  (unsigned)content->cookie);
00294     return(ans);
00295 }
00296 
00297 PUBLIC int
00298 r_store_name_append_components(struct ccn_charbuf *dst,
00299                                struct ccnr_handle *h,
00300                                struct content_entry *content,
00301                                int skip,
00302                                int count)
00303 {
00304     int res;
00305     
00306     res = ccn_name_append_flatname(dst,
00307                                    content->flatname->buf,
00308                                    content->flatname->length, skip, count);
00309     return(res);
00310 }
00311 
00312 PUBLIC int
00313 r_store_content_flags(struct content_entry *content)
00314 {
00315     return(content->flags);
00316 }
00317 
00318 PUBLIC int
00319 r_store_content_change_flags(struct content_entry *content, int set, int clear)
00320 {
00321     int old = content->flags;
00322     content->flags |= set;
00323     content->flags &= ~clear;
00324     return(old);
00325 }
00326 
00327 /**
00328  * Write a file named index/stable that contains the size of
00329  * repoFile1 when the repository is shut down.
00330  */
00331 static int
00332 r_store_write_stable_point(struct ccnr_handle *h)
00333 {
00334     struct ccn_charbuf *path = NULL;
00335     struct ccn_charbuf *cb = NULL;
00336     int fd;
00337     
00338     path = ccn_charbuf_create();
00339     cb = ccn_charbuf_create();
00340     ccn_charbuf_putf(path, "%s/index/stable", h->directory);
00341     unlink(ccn_charbuf_as_string(path)); /* Should not exist, but just in case. */
00342     fd = open(ccn_charbuf_as_string(path),
00343               O_CREAT | O_EXCL | O_WRONLY | O_TRUNC, 0666);
00344     if (fd == -1) {
00345         ccnr_msg(h, "cannot write stable mark %s: %s",
00346                  ccn_charbuf_as_string(path), strerror(errno));
00347         unlink(ccn_charbuf_as_string(path));
00348     }
00349     else {
00350         ccn_charbuf_putf(cb, "%ju", (uintmax_t)(h->stable));
00351         write(fd, cb->buf, cb->length);
00352         close(fd);
00353         if (CCNSHOULDLOG(h, dfsdf, CCNL_INFO))
00354             ccnr_msg(h, "Index marked stable - %s", ccn_charbuf_as_string(cb));
00355     }
00356     ccn_charbuf_destroy(&path);
00357     ccn_charbuf_destroy(&cb);
00358     return(0);
00359 }
00360 
00361 /**
00362  * Read the former size of repoFile1 from index/stable, and remove
00363  * the latter.
00364  */
00365 static void
00366 r_store_read_stable_point(struct ccnr_handle *h)
00367 {
00368     struct ccn_charbuf *path = NULL;
00369     struct ccn_charbuf *cb = NULL;
00370     int fd;
00371     int i;
00372     ssize_t rres;
00373     uintmax_t val;
00374     unsigned char c;
00375     
00376     path = ccn_charbuf_create();
00377     cb = ccn_charbuf_create();
00378     ccn_charbuf_putf(path, "%s/index/stable", h->directory);
00379     fd = open(ccn_charbuf_as_string(path), O_RDONLY, 0666);
00380     if (fd != -1) {
00381         rres = read(fd, ccn_charbuf_reserve(cb, 80), 80);
00382         if (rres > 0)
00383             cb->length = rres;
00384         close(fd);
00385         if (CCNSHOULDLOG(h, dfsdf, CCNL_INFO))
00386             ccnr_msg(h, "Last stable at %s", ccn_charbuf_as_string(cb));
00387     }
00388     for (val = 0, i = 0; i < cb->length; i++) {
00389         c = cb->buf[i];
00390         if ('0' <= c && c <= '9')
00391             val = val * 10 + (c - '0');
00392         else
00393             break;
00394     }
00395     if (i == 0 || i < cb->length) {
00396         ccnr_msg(h, "Bad stable mark - %s", ccn_charbuf_as_string(cb));
00397         h->stable = 0;
00398     }
00399     else {
00400         h->stable = val;
00401         unlink(ccn_charbuf_as_string(path));
00402     }
00403     ccn_charbuf_destroy(&path);
00404     ccn_charbuf_destroy(&cb);
00405 }
00406 
00407 /**
00408  * Log a bit if we are taking a while to re-index.
00409  */
00410 static int
00411 r_store_reindexing(struct ccn_schedule *sched,
00412                    void *clienth,
00413                    struct ccn_scheduled_event *ev,
00414                    int flags)
00415 {
00416     struct ccnr_handle *h = clienth;
00417     struct fdholder *in = NULL;
00418     unsigned pct;
00419     
00420     if ((flags & CCN_SCHEDULE_CANCEL) != 0)
00421         return(0);
00422     in = r_io_fdholder_from_fd(h, h->active_in_fd);
00423     if (in == NULL)
00424         return(0);
00425     pct = ccnr_meter_total(in->meter[FM_BYTI]) / ((h->startupbytes / 100) + 1);
00426     if (pct >= 100)
00427         return(0);
00428     ccnr_msg(h, "indexing %u%% complete", pct);
00429     return(2000000);
00430 }
00431 
00432 /**
00433  * Select power of 2 between l and m + 1 (if possible).
00434  */
00435 static unsigned
00436 choose_limit(unsigned l, unsigned m)
00437 {
00438     unsigned k;
00439     
00440     for (k = 0; k < l; k = 2 * k + 1)
00441         continue;
00442     while (k > (m | 1) || k + 1 < k)
00443         k >>= 1;
00444     return(k + 1);
00445 }
00446 
00447 PUBLIC void
00448 r_store_init(struct ccnr_handle *h)
00449 {
00450     struct ccn_btree *btree = NULL;
00451     struct ccn_btree_node *node = NULL;
00452     struct hashtb_param param = {0};
00453     int i;
00454     int j;
00455     int res;
00456     struct ccn_charbuf *path = NULL;
00457     struct ccn_charbuf *msgs = NULL;
00458     off_t offset;
00459     
00460     path = ccn_charbuf_create();
00461     param.finalize_data = h;
00462     param.finalize = 0;
00463     
00464     h->cob_limit = r_init_confval(h, "CCNR_CONTENT_CACHE", 16, 2000000, 4201);
00465     h->cookie_limit = choose_limit(h->cob_limit, (ccnr_cookie)(~0U));
00466     h->content_by_cookie = calloc(h->cookie_limit, sizeof(h->content_by_cookie[0]));
00467     CHKPTR(h->content_by_cookie);
00468     h->content_by_accession_tab = hashtb_create(sizeof(struct content_by_accession_entry), NULL);
00469     CHKPTR(h->content_by_accession_tab);
00470     h->btree = btree = ccn_btree_create();
00471     CHKPTR(btree);
00472     FAILIF(btree->nextnodeid != 1);
00473     ccn_charbuf_putf(path, "%s/index", h->directory);
00474     res = mkdir(ccn_charbuf_as_string(path), 0700);
00475     if (res != 0 && errno != EEXIST)
00476         r_init_fail(h, __LINE__, ccn_charbuf_as_string(path), errno);
00477     else {
00478         msgs = ccn_charbuf_create();
00479         btree->io = ccn_btree_io_from_directory(ccn_charbuf_as_string(path), msgs);
00480         if (btree->io == NULL)
00481             res = errno;
00482         if (msgs->length != 0 && CCNSHOULDLOG(h, sffdsdf, CCNL_WARNING)) {
00483             ccnr_msg(h, "while initializing %s - %s",
00484                      ccn_charbuf_as_string(path),
00485                      ccn_charbuf_as_string(msgs));
00486         }
00487         ccn_charbuf_destroy(&msgs);
00488         if (btree->io == NULL)
00489             r_init_fail(h, __LINE__, ccn_charbuf_as_string(path), res);
00490     }
00491     node = ccn_btree_getnode(btree, 1, 0);
00492     if (btree->io != NULL)
00493         btree->nextnodeid = btree->io->maxnodeid + 1;
00494     CHKPTR(node);
00495     if (node->buf->length == 0) {
00496         res = ccn_btree_init_node(node, 0, 'R', 0);
00497         CHKSYS(res);
00498     }
00499     ccn_charbuf_destroy(&path);
00500     if (h->running == -1)
00501         return;
00502     r_store_read_stable_point(h);
00503     h->active_in_fd = -1;
00504     h->active_out_fd = r_io_open_repo_data_file(h, "repoFile1", 1); /* output */
00505     offset = lseek(h->active_out_fd, 0, SEEK_END);
00506     h->startupbytes = offset;
00507     if (offset != h->stable || node->corrupt != 0) {
00508         ccnr_msg(h, "Index not current - resetting");
00509         ccn_btree_init_node(node, 0, 'R', 0);
00510         node = NULL;
00511         ccn_btree_destroy(&h->btree);
00512         path = ccn_charbuf_create();
00513         /* Remove old index files to avoid confusion */
00514         for (i = 1, j = 0; i > 0 && j < 3; i++) {
00515             path->length = 0;
00516             res = ccn_charbuf_putf(path, "%s/index/%d", h->directory, i);
00517             if (res >= 0)
00518                 res = unlink(ccn_charbuf_as_string(path));
00519             if (res < 0)
00520                 j++;
00521         }
00522         h->btree = btree = ccn_btree_create();
00523         path->length = 0;
00524         ccn_charbuf_putf(path, "%s/index", h->directory);
00525         btree->io = ccn_btree_io_from_directory(ccn_charbuf_as_string(path), msgs);
00526         CHKPTR(btree->io);
00527         btree->io->maxnodeid = 0;
00528         btree->nextnodeid = 1;
00529         node = ccn_btree_getnode(btree, 1, 0);
00530         btree->nextnodeid = btree->io->maxnodeid + 1;
00531         ccn_btree_init_node(node, 0, 'R', 0);
00532         h->stable = 0;
00533         h->active_in_fd = r_io_open_repo_data_file(h, "repoFile1", 0); /* input */
00534         ccn_charbuf_destroy(&path);
00535         if (CCNSHOULDLOG(h, dfds, CCNL_INFO))
00536             ccn_schedule_event(h->sched, 50000, r_store_reindexing, NULL, 0);
00537     }
00538     if (CCNSHOULDLOG(h, weuyg, CCNL_FINEST)) {
00539         FILE *dumpfile = NULL;
00540         
00541         path = ccn_charbuf_create();
00542         ccn_charbuf_putf(path, "%s/index/btree_check.out", h->directory);
00543         dumpfile = fopen(ccn_charbuf_as_string(path), "w");
00544         res = ccn_btree_check(btree, dumpfile);
00545         if (dumpfile != NULL) {
00546             fclose(dumpfile);
00547             dumpfile = NULL;
00548         }
00549         else
00550             path->length = 0;
00551         ccnr_msg(h, "ccn_btree_check returned %d (%s)",
00552                     res, ccn_charbuf_as_string(path));
00553         ccn_charbuf_destroy(&path);
00554         if (res < 0)
00555             r_init_fail(h, __LINE__, "index is corrupt", res);
00556     }
00557     btree->full = r_init_confval(h, "CCNR_BTREE_MAX_FANOUT", 4, 9999, 1999);
00558     btree->full0 = r_init_confval(h, "CCNR_BTREE_MAX_LEAF_ENTRIES", 4, 9999, 1999);
00559     btree->nodebytes = r_init_confval(h, "CCNR_BTREE_MAX_NODE_BYTES", 1024, 8388608, 2097152);
00560     btree->nodepool = r_init_confval(h, "CCNR_BTREE_NODE_POOL", 16, 2000000, 512);
00561     if (h->running != -1)
00562         r_store_index_needs_cleaning(h);
00563 }
00564 
00565 PUBLIC int
00566 r_store_final(struct ccnr_handle *h, int stable) {
00567     int res;
00568     
00569     res = ccn_btree_destroy(&h->btree);
00570     if (res < 0)
00571         ccnr_msg(h, "r_store_final.%d-%d Errors while closing index", __LINE__, res);
00572     if (res >= 0 && stable)
00573         res = r_store_write_stable_point(h);
00574     return(res);
00575 }
00576     
00577 PUBLIC struct content_entry *
00578 r_store_content_from_accession(struct ccnr_handle *h, ccnr_accession accession)
00579 {
00580     struct ccn_parsed_ContentObject obj = {0};
00581     struct content_entry *content = NULL;
00582     struct content_by_accession_entry *entry;
00583     const unsigned char *content_base = NULL;
00584     int res;
00585     ccnr_accession acc;
00586     
00587     if (accession == CCNR_NULL_ACCESSION)
00588         return(NULL);
00589     entry = hashtb_lookup(h->content_by_accession_tab,
00590                           &accession, sizeof(accession));
00591     if (entry != NULL) {
00592         h->content_from_accession_hits++;
00593         return(entry->content);
00594     }
00595     h->content_from_accession_misses++;
00596     content = calloc(1, sizeof(*content));
00597     CHKPTR(content);
00598     content->cookie = 0;
00599     content->accession = accession;
00600     content->cob = NULL;
00601     content->size = 0;
00602     content_base = r_store_content_base(h, content);
00603     if (content_base == NULL || content->size == 0)
00604         goto Bail;
00605     res = r_store_set_flatname(h, content, &obj);
00606     if (res < 0) goto Bail;
00607     r_store_enroll_content(h, content);
00608     res = r_store_content_btree_insert(h, content, &obj, &acc);
00609     if (res < 0) goto Bail;
00610     if (res == 1 || CCNSHOULDLOG(h, sdf, CCNL_FINEST))
00611         ccnr_debug_content(h, __LINE__, "content/accession", NULL, content);
00612     return(content);
00613 Bail:
00614     ccnr_msg(h, "r_store_content_from_accession.%d failed 0x%jx",
00615              __LINE__, ccnr_accession_encode(h, accession));
00616     r_store_forget_content(h, &content);
00617     return(content);
00618 }
00619 
00620 PUBLIC struct content_entry *
00621 r_store_content_from_cookie(struct ccnr_handle *h, ccnr_cookie cookie)
00622 {
00623     struct content_entry *ans = NULL;
00624     
00625     ans = h->content_by_cookie[cookie & (h->cookie_limit - 1)];
00626     if (ans != NULL && ans->cookie != cookie)
00627         ans = NULL;
00628     return(ans);
00629 }
00630 
00631 /**
00632  * This makes a cookie for content, and, if it has an accession number already,
00633  * enters it into the content_by_accession_tab.  Does not index by name.
00634  */
00635 PUBLIC ccnr_cookie
00636 r_store_enroll_content(struct ccnr_handle *h, struct content_entry *content)
00637 {
00638     ccnr_cookie cookie;
00639     unsigned mask;
00640     
00641     mask = h->cookie_limit - 1;
00642     cookie = ++(h->cookie);
00643     if (cookie == 0)
00644         cookie = ++(h->cookie); /* Cookie numbers may wrap */
00645     // XXX - check for persistence here, if we add that
00646     r_store_forget_content(h, &(h->content_by_cookie[cookie & mask]));
00647     content->cookie = cookie;
00648     h->content_by_cookie[cookie & mask] = content;
00649     if (content->accession != CCNR_NULL_ACCESSION) {
00650         struct hashtb_enumerator ee;
00651         struct hashtb_enumerator *e = &ee;
00652         ccnr_accession accession = content->accession;
00653         struct content_by_accession_entry *entry = NULL;
00654         hashtb_start(h->content_by_accession_tab, e);
00655         hashtb_seek(e, &accession, sizeof(accession), 0);
00656         entry = e->data;
00657         if (entry != NULL)
00658             entry->content = content;
00659         hashtb_end(e);
00660         content->flags |= CCN_CONTENT_ENTRY_STABLE;
00661     }
00662     return(cookie);
00663 }
00664 
00665 /** @returns 2 if content was added to index, 1 if it was there but had no accession, 0 if it was already there, -1 for error */
00666 static int
00667 r_store_content_btree_insert(struct ccnr_handle *h,
00668                              struct content_entry *content,
00669                              struct ccn_parsed_ContentObject *pco,
00670                              ccnr_accession *accp)
00671 {
00672     const unsigned char *content_base = NULL;
00673     struct ccn_btree *btree = NULL;
00674     struct ccn_btree_node *leaf = NULL;
00675     struct ccn_btree_node *node = NULL;
00676     struct ccn_charbuf *flat = NULL;
00677     int i;
00678     int limit;
00679     int res;
00680 
00681     btree = h->btree;
00682     if (btree == NULL)
00683         return(-1);
00684     flat = content->flatname;
00685     if (flat == NULL)
00686         return(-1);
00687     res = ccn_btree_lookup(h->btree, flat->buf, flat->length, &leaf);
00688     if (res < 0)
00689         return(-1);
00690     i = CCN_BT_SRCH_INDEX(res);
00691     if (CCN_BT_SRCH_FOUND(res)) {
00692         *accp = ccnr_accession_decode(h, ccn_btree_content_cobid(leaf, i));
00693         return(*accp == CCNR_NULL_ACCESSION);
00694     }
00695     else {
00696         content_base = r_store_content_base(h, content);
00697         if (content_base == NULL)
00698             return(-1);
00699         res = ccn_btree_prepare_for_update(h->btree, leaf);
00700         if (res < 0)
00701             return(-1);
00702         res = ccn_btree_insert_content(leaf, i,
00703                                        ccnr_accession_encode(h, content->accession),
00704                                        content_base,
00705                                        pco,
00706                                        content->flatname);
00707         if (res < 0)
00708             return(-1);
00709         if (ccn_btree_oversize(btree, leaf)) {
00710             res = ccn_btree_split(btree, leaf);
00711             for (limit = 100; res >= 0 && btree->nextsplit != 0; limit--) {
00712                 if (limit == 0) abort();
00713                 node = ccn_btree_getnode(btree, btree->nextsplit, 0);
00714                 if (node == NULL)
00715                     return(-1);
00716                 res = ccn_btree_split(btree, node);
00717             }
00718         }
00719         r_store_index_needs_cleaning(h);
00720         
00721         *accp = content->accession;
00722         return(2);
00723     }
00724 }
00725 
00726 /**
00727  *  Remove internal representation of a content object
00728  */
00729 PUBLIC void
00730 r_store_forget_content(struct ccnr_handle *h, struct content_entry **pentry)
00731 {
00732     unsigned i;
00733     struct content_entry *entry = *pentry;
00734     
00735     if (entry == NULL)
00736         return;
00737     *pentry = NULL;
00738     if ((entry->flags & CCN_CONTENT_ENTRY_STALE) != 0)
00739         h->n_stale--;
00740     if (CCNSHOULDLOG(h, LM_4, CCNL_FINER))
00741         ccnr_debug_content(h, __LINE__, "remove", NULL, entry);
00742     /* Remove the cookie reference */
00743     i = entry->cookie & (h->cookie_limit - 1);
00744     if (h->content_by_cookie[i] == entry)
00745         h->content_by_cookie[i] = NULL;
00746     entry->cookie = 0;
00747     /* Remove the accession reference */
00748     if (entry->accession != CCNR_NULL_ACCESSION) {
00749         struct hashtb_enumerator ee;
00750         struct hashtb_enumerator *e = &ee;
00751         hashtb_start(h->content_by_accession_tab, e);
00752         if (hashtb_seek(e, &entry->accession, sizeof(entry->accession), 0) ==
00753             HT_NEW_ENTRY) {
00754             ccnr_msg(h, "orphaned content %llu",
00755                      (unsigned long long)(entry->accession));
00756             hashtb_delete(e);
00757             hashtb_end(e);
00758             return;
00759         }
00760         hashtb_delete(e);
00761         hashtb_end(e);
00762         entry->accession = CCNR_NULL_ACCESSION;
00763     }
00764     /* Clean up allocated subfields */
00765     ccn_charbuf_destroy(&entry->flatname);
00766     if (entry->cob != NULL) {
00767         h->cob_count--;
00768         ccn_charbuf_destroy(&entry->cob);
00769     }
00770     free(entry);
00771 }
00772 
00773 /**
00774  *  Get a handle on the content object that matches key, or if there is
00775  * no match, the one that would come just after it.
00776  *
00777  * The key is in flatname format.
00778  */
00779 static struct content_entry *    
00780 r_store_look(struct ccnr_handle *h, const unsigned char *key, size_t size)
00781 {
00782     struct content_entry *content = NULL;
00783     struct ccn_btree_node *leaf = NULL;
00784     ccnr_accession accession;
00785     int ndx;
00786     int res;
00787 
00788     res = ccn_btree_lookup(h->btree, key, size, &leaf);
00789     if (res >= 0) {
00790         ndx = CCN_BT_SRCH_INDEX(res);
00791         if (ndx == ccn_btree_node_nent(leaf)) {
00792             res = ccn_btree_next_leaf(h->btree, leaf, &leaf);
00793             if (res <= 0)
00794                 return(NULL);
00795             ndx = 0;
00796         }
00797         accession = ccnr_accession_decode(h, ccn_btree_content_cobid(leaf, ndx));
00798         if (accession != CCNR_NULL_ACCESSION) {
00799             struct content_by_accession_entry *entry;
00800             entry = hashtb_lookup(h->content_by_accession_tab,
00801                                     &accession, sizeof(accession));
00802             if (entry != NULL)
00803                 content = entry->content;
00804             if (content == NULL) {
00805                 /* Construct handle without actually reading the cob */
00806                 res = ccn_btree_content_cobsz(leaf, ndx);
00807                 content = calloc(1, sizeof(*content));
00808                 if (res > 0 && content != NULL) {
00809                     content->accession = accession;
00810                     content->cob = NULL;
00811                     content->size = res;
00812                     content->flatname = ccn_charbuf_create();
00813                     CHKPTR(content->flatname);
00814                     res = ccn_btree_key_fetch(content->flatname, leaf, ndx);
00815                     CHKRES(res);
00816                     r_store_enroll_content(h, content);
00817                 }
00818             }
00819         }
00820     }
00821     return(content);
00822 }
00823 
00824 PUBLIC struct content_entry *
00825 r_store_find_first_match_candidate(struct ccnr_handle *h,
00826                                    const unsigned char *interest_msg,
00827                                    const struct ccn_parsed_interest *pi)
00828 {
00829     int res;
00830     size_t start = pi->offset[CCN_PI_B_Name];
00831     size_t end = pi->offset[CCN_PI_E_Name];
00832     struct ccn_charbuf *namebuf = NULL;
00833     struct ccn_charbuf *flatname = NULL;
00834     struct content_entry *content = NULL;
00835     
00836     flatname = ccn_charbuf_create();
00837     ccn_flatname_from_ccnb(flatname, interest_msg, pi->offset[CCN_PI_E]);
00838     if (pi->offset[CCN_PI_B_Exclude] < pi->offset[CCN_PI_E_Exclude]) {
00839         /* Check for <Exclude><Any/><Component>... fast case */
00840         struct ccn_buf_decoder decoder;
00841         struct ccn_buf_decoder *d;
00842         size_t ex1start;
00843         size_t ex1end;
00844         d = ccn_buf_decoder_start(&decoder,
00845                                   interest_msg + pi->offset[CCN_PI_B_Exclude],
00846                                   pi->offset[CCN_PI_E_Exclude] -
00847                                   pi->offset[CCN_PI_B_Exclude]);
00848         ccn_buf_advance(d);
00849         if (ccn_buf_match_dtag(d, CCN_DTAG_Any)) {
00850             ccn_buf_advance(d);
00851             ccn_buf_check_close(d);
00852             if (ccn_buf_match_dtag(d, CCN_DTAG_Component)) {
00853                 ex1start = pi->offset[CCN_PI_B_Exclude] + d->decoder.token_index;
00854                 ccn_buf_advance_past_element(d);
00855                 ex1end = pi->offset[CCN_PI_B_Exclude] + d->decoder.token_index;
00856                 if (d->decoder.state >= 0) {
00857                     namebuf = ccn_charbuf_create();
00858                     ccn_charbuf_append(namebuf,
00859                                        interest_msg + start,
00860                                        end - start);
00861                     namebuf->length--;
00862                     ccn_charbuf_append(namebuf,
00863                                        interest_msg + ex1start,
00864                                        ex1end - ex1start);
00865                     ccn_charbuf_append_closer(namebuf);
00866                     res = ccn_flatname_append_from_ccnb(flatname,
00867                                                         interest_msg + ex1start,
00868                                                         ex1end - ex1start,
00869                                                         0, 1);
00870                     if (res != 1)
00871                         ccnr_debug_ccnb(h, __LINE__, "fastex_bug", NULL,
00872                                         namebuf->buf, namebuf->length);
00873                     if (CCNSHOULDLOG(h, LM_8, CCNL_FINER))
00874                         ccnr_debug_ccnb(h, __LINE__, "fastex", NULL,
00875                                         namebuf->buf, namebuf->length);
00876                 }
00877             }
00878         }
00879     }
00880     content = r_store_look(h, flatname->buf, flatname->length);
00881     ccn_charbuf_destroy(&namebuf);
00882     ccn_charbuf_destroy(&flatname);
00883     return(content);
00884 }
00885 
00886 PUBLIC int
00887 r_store_content_matches_interest_prefix(struct ccnr_handle *h,
00888                                 struct content_entry *content,
00889                                 const unsigned char *interest_msg,
00890                                 size_t interest_size)
00891 {
00892     struct ccn_charbuf *flatname = ccn_charbuf_create();
00893     int ans;
00894     int cmp;
00895 
00896     ccn_flatname_from_ccnb(flatname, interest_msg, interest_size);
00897     cmp = ccn_flatname_charbuf_compare(flatname, content->flatname);
00898     ans = (cmp == 0 || cmp == -9999);
00899     ccn_charbuf_destroy(&flatname);
00900     return(ans);
00901 }
00902 
00903 PUBLIC struct content_entry *
00904 r_store_content_next(struct ccnr_handle *h, struct content_entry *content)
00905 {
00906     if (content == NULL)
00907         return(0);
00908     /* We need to go past the current name, so make sure there is a 0 byte */
00909     ccn_charbuf_as_string(content->flatname);
00910     content = r_store_look(h, content->flatname->buf, content->flatname->length + 1);
00911     return(content);
00912 }
00913 
00914 PUBLIC struct content_entry *
00915 r_store_next_child_at_level(struct ccnr_handle *h,
00916                     struct content_entry *content, int level)
00917 {
00918     struct content_entry *next = NULL;
00919     struct ccn_charbuf *name;
00920     struct ccn_charbuf *flatname = NULL;
00921     int res;
00922     
00923     if (content == NULL)
00924         return(NULL);
00925     name = ccn_charbuf_create();
00926     ccn_name_init(name);
00927     res = ccn_name_append_flatname(name,
00928                                    content->flatname->buf,
00929                                    content->flatname->length, 0, level + 1);
00930     if (res < level)
00931         goto Bail;
00932     if (res == level)
00933         res = ccn_name_append(name, NULL, 0);
00934     else if (res == level + 1)
00935         res = ccn_name_next_sibling(name); // XXX - would be nice to have a flatname version of this
00936     if (res < 0)
00937         goto Bail;
00938     if (CCNSHOULDLOG(h, LM_8, CCNL_FINER))
00939         ccnr_debug_ccnb(h, __LINE__, "child_successor", NULL,
00940                         name->buf, name->length);
00941     flatname = ccn_charbuf_create();
00942     ccn_flatname_from_ccnb(flatname, name->buf, name->length);
00943     next = r_store_look(h, flatname->buf, flatname->length);
00944     if (next == content) {
00945         // XXX - I think this case should not occur, but just in case, avoid a loop.
00946         ccnr_debug_content(h, __LINE__, "urp", NULL, next);
00947         next = NULL;
00948     }
00949 Bail:
00950     ccn_charbuf_destroy(&name);
00951     ccn_charbuf_destroy(&flatname);
00952     return(next);
00953 }
00954 
00955 PUBLIC struct content_entry *
00956 r_store_lookup(struct ccnr_handle *h,
00957                const unsigned char *msg,
00958                const struct ccn_parsed_interest *pi,
00959                struct ccn_indexbuf *comps)
00960 {
00961     struct content_entry *content = NULL;
00962     struct ccn_btree_node *leaf = NULL;
00963     ccnr_cookie last_match = 0;
00964     ccnr_accession last_match_acc = CCNR_NULL_ACCESSION;
00965     struct ccn_charbuf *scratch = NULL;
00966     size_t size = pi->offset[CCN_PI_E];
00967     int ndx;
00968     int res;
00969     int try;
00970     
00971     content = r_store_find_first_match_candidate(h, msg, pi);
00972     if (content != NULL && CCNSHOULDLOG(h, LM_8, CCNL_FINER))
00973         ccnr_debug_content(h, __LINE__, "first_candidate", NULL,
00974                            content);
00975     if (content != NULL &&
00976         !r_store_content_matches_interest_prefix(h, content, msg, size)) {
00977             if (CCNSHOULDLOG(h, LM_8, CCNL_FINER))
00978                 ccnr_debug_ccnb(h, __LINE__, "prefix_mismatch", NULL,
00979                                 msg, size);
00980             content = NULL;
00981         }
00982     scratch = ccn_charbuf_create();
00983     for (try = 0; content != NULL; try++) {
00984         res = ccn_btree_lookup(h->btree,
00985                                content->flatname->buf,
00986                                content->flatname->length,
00987                                &leaf);
00988         if (CCN_BT_SRCH_FOUND(res) == 0) {
00989             ccnr_debug_content(h, __LINE__, "impossible", NULL, content);
00990             content = NULL;
00991             break;
00992         }
00993         ndx = CCN_BT_SRCH_INDEX(res);
00994         res = ccn_btree_match_interest(leaf, ndx, msg, pi, scratch);
00995         if (res == -1) {
00996             ccnr_debug_ccnb(h, __LINE__, "match_error", NULL, msg, size);
00997             content = NULL;
00998             break;
00999         }
01000         if (res == 1) {
01001             if ((pi->orderpref & 1) == 0) // XXX - should be symbolic
01002                 break;
01003             last_match = content->cookie;
01004             last_match_acc = content->accession;
01005             content = r_store_next_child_at_level(h, content, comps->n - 1);
01006         }
01007         else
01008             content = r_store_content_next(h, content);
01009         if (content != NULL &&
01010             !r_store_content_matches_interest_prefix(h, content, msg, size))
01011                 content = NULL;
01012     }
01013     if (last_match != 0) {
01014         content = r_store_content_from_cookie(h, last_match);
01015         if (content == NULL)
01016             content = r_store_content_from_accession(h, last_match_acc);
01017     }
01018     ccn_charbuf_destroy(&scratch);
01019     return(content);
01020 }
01021 
01022 /**
01023  * Find the first content handle that matches the prefix given by the namish,
01024  * which may be a Name, Interest, ContentObject, ...
01025  *
01026  * Does not check the other parts of namish, in particular, does not generate
01027  * the digest component of a ContentObject.
01028  */
01029 PUBLIC struct content_entry *
01030 r_store_lookup_ccnb(struct ccnr_handle *h,
01031                     const unsigned char *namish, size_t size)
01032 {
01033     struct content_entry *content = NULL;
01034     struct ccn_charbuf *flatname = NULL;
01035     int res;
01036     
01037     flatname = ccn_charbuf_create();
01038     if (flatname == NULL)
01039         goto Bail;
01040     res = ccn_flatname_from_ccnb(flatname, namish, size);
01041     if (res < 0)
01042         goto Bail;
01043     content = r_store_look(h, flatname->buf, flatname->length);
01044     if (content != NULL) {
01045         res = ccn_flatname_charbuf_compare(flatname, content->flatname);
01046         if (res == 0 || res == -9999) {
01047             /* prefix matches */
01048         }
01049         else
01050             content = NULL;
01051     }
01052 Bail:
01053     ccn_charbuf_destroy(&flatname);
01054     return(content);
01055 }
01056 
01057 /**
01058  * Mark content as stale
01059  */
01060 PUBLIC void
01061 r_store_mark_stale(struct ccnr_handle *h, struct content_entry *content)
01062 {
01063     ccnr_cookie cookie = content->cookie;
01064     if ((content->flags & CCN_CONTENT_ENTRY_STALE) != 0)
01065         return;
01066     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01067             ccnr_debug_content(h, __LINE__, "stale", NULL, content);
01068     content->flags |= CCN_CONTENT_ENTRY_STALE;
01069     h->n_stale++;
01070     if (cookie < h->min_stale)
01071         h->min_stale = cookie;
01072     if (cookie > h->max_stale)
01073         h->max_stale = cookie;
01074 }
01075 
01076 /**
01077  * Scheduled event that makes content stale when its FreshnessSeconds
01078  * has expired.
01079  */
01080 static int
01081 expire_content(struct ccn_schedule *sched,
01082                void *clienth,
01083                struct ccn_scheduled_event *ev,
01084                int flags)
01085 {
01086     struct ccnr_handle *h = clienth;
01087     ccnr_cookie cookie = ev->evint;
01088     struct content_entry *content = NULL;
01089     if ((flags & CCN_SCHEDULE_CANCEL) != 0)
01090         return(0);
01091     content = r_store_content_from_cookie(h, cookie);
01092     if (content != NULL)
01093         r_store_mark_stale(h, content);
01094     return(0);
01095 }
01096 
01097 /**
01098  * Schedules content expiration based on its FreshnessSeconds.
01099  *
01100  */
01101 PUBLIC void
01102 r_store_set_content_timer(struct ccnr_handle *h, struct content_entry *content,
01103                   struct ccn_parsed_ContentObject *pco)
01104 {
01105     int seconds = 0;
01106     int microseconds = 0;
01107     size_t start = pco->offset[CCN_PCO_B_FreshnessSeconds];
01108     size_t stop  = pco->offset[CCN_PCO_E_FreshnessSeconds];
01109     const unsigned char *content_msg = NULL;
01110     if (start == stop)
01111         return;
01112     content_msg = r_store_content_base(h, content);
01113     seconds = ccn_fetch_tagged_nonNegativeInteger(
01114                 CCN_DTAG_FreshnessSeconds,
01115                 content_msg,
01116                 start, stop);
01117     if (seconds <= 0)
01118         return;
01119     if (seconds > ((1U<<31) / 1000000)) {
01120         ccnr_debug_content(h, __LINE__, "FreshnessSeconds_too_large", NULL,
01121                            content);
01122         return;
01123     }
01124     microseconds = seconds * 1000000;
01125     ccn_schedule_event(h->sched, microseconds,
01126                        &expire_content, NULL, content->cookie);
01127 }
01128 
01129 /**
01130  * Parses content object and sets content->flatname
01131  */
01132 static int
01133 r_store_set_flatname(struct ccnr_handle *h, struct content_entry *content,
01134                      struct ccn_parsed_ContentObject *pco)
01135 {
01136     int res;
01137     struct ccn_charbuf *flatname = NULL;
01138     const unsigned char *msg = NULL;
01139     size_t size;
01140     
01141     msg = r_store_content_base(h, content);
01142     size = content->size;
01143     if (msg == NULL)
01144         goto Bail;
01145     flatname = ccn_charbuf_create();
01146     if (flatname == NULL)
01147         goto Bail;    
01148     res = ccn_parse_ContentObject(msg, size, pco, NULL);
01149     if (res < 0) {
01150         ccnr_msg(h, "error parsing ContentObject - code %d", res);
01151         goto Bail;
01152     }
01153     ccn_digest_ContentObject(msg, pco);
01154     if (pco->digest_bytes != 32)
01155         goto Bail;
01156     res = ccn_flatname_from_ccnb(flatname, msg, size);
01157     if (res < 0) goto Bail;
01158     res = ccn_flatname_append_component(flatname, pco->digest, pco->digest_bytes);
01159     if (res < 0) goto Bail;
01160     content->flatname = flatname;
01161     flatname = NULL;
01162     return(0);
01163 Bail:
01164     ccn_charbuf_destroy(&flatname);
01165     return(-1);
01166 }
01167 
01168 /**
01169  *  Get the flatname associated with content
01170  *
01171  * @returns flatname in a charbuf, which should be treated as read-only.
01172  */
01173 PUBLIC struct ccn_charbuf *
01174 r_store_content_flatname(struct ccnr_handle *h, struct content_entry *content)
01175 {
01176     return(content->flatname);
01177 }
01178 
01179 PUBLIC struct content_entry *
01180 process_incoming_content(struct ccnr_handle *h, struct fdholder *fdholder,
01181                          unsigned char *msg, size_t size)
01182 {
01183     struct ccn_parsed_ContentObject obj = {0};
01184     int res;
01185     struct content_entry *content = NULL;
01186     ccnr_accession accession = CCNR_NULL_ACCESSION;
01187     
01188     content = calloc(1, sizeof(*content));
01189     if (content == NULL)
01190         goto Bail;    
01191     content->cob = ccn_charbuf_create();
01192     if (content->cob == NULL)
01193         goto Bail;    
01194     res = ccn_charbuf_append(content->cob, msg, size);
01195     if (res < 0) goto Bail;
01196     content->size = size;
01197     res = r_store_set_flatname(h, content, &obj);
01198     if (res < 0) goto Bail;
01199     ccnr_meter_bump(h, fdholder->meter[FM_DATI], 1);
01200     content->accession = CCNR_NULL_ACCESSION;
01201     r_store_enroll_content(h, content);
01202     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01203         ccnr_debug_content(h, __LINE__, "content_from", fdholder, content);
01204     res = r_store_content_btree_insert(h, content, &obj, &accession);
01205     if (res < 0) goto Bail;
01206     if (res == 0) {
01207         /* Content was there, with an accession */
01208         if (CCNSHOULDLOG(h, LM_4, CCNL_FINER))
01209             ccnr_debug_content(h, __LINE__, "content_duplicate",
01210                                fdholder, content);
01211         h->content_dups_recvd++;
01212         r_store_forget_content(h, &content);
01213         content = r_store_content_from_accession(h, accession);
01214         if (content == NULL)
01215             goto Bail;
01216     }
01217     r_store_set_content_timer(h, content, &obj);
01218     r_match_match_interests(h, content, &obj, NULL, fdholder);
01219     return(content);
01220 Bail:
01221     r_store_forget_content(h, &content);
01222     return(content);
01223 }
01224 
01225 PUBLIC int
01226 r_store_content_field_access(struct ccnr_handle *h,
01227                              struct content_entry *content,
01228                              enum ccn_dtag dtag,
01229                              const unsigned char **bufp, size_t *sizep)
01230 {
01231     int res = -1;
01232     const unsigned char *content_msg;
01233     struct ccn_parsed_ContentObject pco = {0};
01234     
01235     content_msg = r_store_content_base(h, content);
01236     if (content_msg == NULL)
01237         return(-1);
01238     res = ccn_parse_ContentObject(content_msg, content->size, &pco, NULL);
01239     if (res < 0)
01240         return(-1);
01241     if (dtag == CCN_DTAG_Content)
01242         res = ccn_ref_tagged_BLOB(CCN_DTAG_Content, content_msg,
01243                                   pco.offset[CCN_PCO_B_Content],
01244                                   pco.offset[CCN_PCO_E_Content],
01245                                   bufp, sizep);
01246     return(res);
01247 }
01248 
01249 const ccnr_accession r_store_mark_repoFile1 = ((ccnr_accession)1) << 48;
01250 
01251 PUBLIC int
01252 r_store_set_accession_from_offset(struct ccnr_handle *h,
01253                                   struct content_entry *content,
01254                                   struct fdholder *fdholder, off_t offset)
01255 {
01256     struct ccn_btree_node *leaf = NULL;
01257     uint_least64_t cobid;
01258     int ndx;
01259     int res = -1;
01260     
01261     if (offset != (off_t)-1 && content->accession == CCNR_NULL_ACCESSION) {
01262         struct hashtb_enumerator ee;
01263         struct hashtb_enumerator *e = &ee;
01264         struct content_by_accession_entry *entry = NULL;
01265         
01266         content->flags |= CCN_CONTENT_ENTRY_STABLE;
01267         content->accession = ((ccnr_accession)offset) | r_store_mark_repoFile1;
01268         hashtb_start(h->content_by_accession_tab, e);
01269         hashtb_seek(e, &content->accession, sizeof(content->accession), 0);
01270         entry = e->data;
01271         if (entry != NULL) {
01272             entry->content = content;
01273             if (content->cob != NULL)
01274                 h->cob_count++;
01275         }
01276         hashtb_end(e);
01277         if (content->flatname != NULL) {
01278             res = ccn_btree_lookup(h->btree,
01279                                    content->flatname->buf,
01280                                    content->flatname->length, &leaf);
01281             if (res >= 0 && CCN_BT_SRCH_FOUND(res)) {
01282                 ndx = CCN_BT_SRCH_INDEX(res);
01283                 cobid = ccnr_accession_encode(h, content->accession);
01284                 ccn_btree_prepare_for_update(h->btree, leaf);
01285                 res = ccn_btree_content_set_cobid(leaf, ndx, cobid);
01286             }
01287             else
01288                 res = -1;
01289         }
01290         if (res >= 0 && content->accession >= h->notify_after) 
01291             r_sync_notify_content(h, 0, content);
01292     }
01293     return(res);
01294 }
01295 
01296 PUBLIC void
01297 r_store_send_content(struct ccnr_handle *h, struct fdholder *fdholder, struct content_entry *content)
01298 {
01299     const unsigned char *content_msg = NULL;
01300     off_t offset;
01301 
01302     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01303         ccnr_debug_content(h, __LINE__, "content_to", fdholder, content);
01304     content_msg = r_store_content_base(h, content);
01305     r_link_stuff_and_send(h, fdholder, content_msg, content->size, NULL, 0, &offset);
01306     if (offset != (off_t)-1 && content->accession == CCNR_NULL_ACCESSION) {
01307         int res;
01308         res = r_store_set_accession_from_offset(h, content, fdholder, offset);
01309         if (res == 0)
01310             if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01311                 ccnr_debug_content(h, __LINE__, "content_stored",
01312                                    r_io_fdholder_from_fd(h, h->active_out_fd),
01313                                    content);
01314     }
01315 }
01316 
01317 PUBLIC int
01318 r_store_commit_content(struct ccnr_handle *h, struct content_entry *content)
01319 {
01320     // XXX - here we need to check if this is something we *should* be storing, according to our policy
01321     if ((r_store_content_flags(content) & CCN_CONTENT_ENTRY_STABLE) == 0) {
01322         r_store_send_content(h, r_io_fdholder_from_fd(h, h->active_out_fd), content);
01323         r_store_content_change_flags(content, CCN_CONTENT_ENTRY_STABLE, 0);
01324     }
01325     return(0);
01326 }
01327 
01328 PUBLIC void
01329 ccnr_debug_content(struct ccnr_handle *h,
01330                    int lineno,
01331                    const char *msg,
01332                    struct fdholder *fdholder,
01333                    struct content_entry *content)
01334 {
01335     struct ccn_charbuf *c = ccn_charbuf_create();
01336     struct ccn_charbuf *flat = content->flatname;
01337     
01338     if (c == NULL)
01339         return;
01340     ccn_charbuf_putf(c, "debug.%d %s ", lineno, msg);
01341     if (fdholder != NULL)
01342         ccn_charbuf_putf(c, "%u ", fdholder->filedesc);
01343     if (flat != NULL)
01344         ccn_uri_append_flatname(c, flat->buf, flat->length, 1);
01345     ccn_charbuf_putf(c, " (%d bytes)", content->size);
01346     ccnr_msg(h, "%s", ccn_charbuf_as_string(c));
01347     ccn_charbuf_destroy(&c);
01348 }
01349 
01350 /** Number of btree index writes to do in a batch */
01351 #define CCN_BT_CLEAN_BATCH 3
01352 /** Approximate delay between batches of btree index writes */
01353 #define CCN_BT_CLEAN_TICK_MICROS 65536
01354 static int
01355 r_store_index_cleaner(struct ccn_schedule *sched,
01356     void *clienth,
01357     struct ccn_scheduled_event *ev,
01358     int flags)
01359 {
01360     struct ccnr_handle *h = clienth;
01361     struct hashtb_enumerator ee;
01362     struct hashtb_enumerator *e = &ee;
01363     struct ccn_btree_node *node = NULL;
01364     int k;
01365     int res;
01366     int overquota;
01367     
01368     (void)(sched);
01369     (void)(ev);
01370     if ((flags & CCN_SCHEDULE_CANCEL) != 0 ||
01371          h->btree == NULL || h->btree->io == NULL) {
01372         h->index_cleaner = NULL;
01373         ccn_indexbuf_destroy(&h->toclean);
01374         return(0);
01375     }
01376     /* First, work on cleaning the things we already know need cleaning */
01377     if (h->toclean != NULL) {
01378         for (k = 0; k < CCN_BT_CLEAN_BATCH && h->toclean->n > 0; k++) {
01379             node = ccn_btree_rnode(h->btree, h->toclean->buf[--h->toclean->n]);
01380             if (node != NULL && node->iodata != NULL) {
01381                 res = ccn_btree_chknode(node); /* paranoia */
01382                 if (res < 0 || CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01383                     ccnr_msg(h, "write index node %u (err %d)",
01384                              (unsigned)node->nodeid, node->corrupt);
01385                 if (res >= 0) {
01386                     if (node->clean != node->buf->length)
01387                         res = h->btree->io->btwrite(h->btree->io, node);
01388                     if (res < 0)
01389                         ccnr_msg(h, "failed to write index node %u",
01390                                  (unsigned)node->nodeid);
01391                     else
01392                         node->clean = node->buf->length;
01393                 }
01394                 if (res >= 0 && node->iodata != NULL && node->activity == 0) {
01395                     if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01396                         ccnr_msg(h, "close index node %u",
01397                                  (unsigned)node->nodeid);
01398                     res = ccn_btree_close_node(h->btree, node);
01399                 }
01400             }
01401         }
01402         if (h->toclean->n > 0)
01403             return(nrand48(h->seed) % (2U * CCN_BT_CLEAN_TICK_MICROS) + 500);
01404     }
01405     /* Sweep though and find the nodes that still need cleaning */
01406     overquota = 0;
01407     if (h->btree->nodepool >= 16)
01408         overquota = hashtb_n(h->btree->resident) - h->btree->nodepool;
01409     hashtb_start(h->btree->resident, e);
01410     for (node = e->data; node != NULL; node = e->data) {
01411         if (overquota > 0 &&
01412               node->activity == 0 &&
01413               node->iodata == NULL &&
01414               node->clean == node->buf->length) {
01415             overquota -= 1;
01416             if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINEST))
01417                 ccnr_msg(h, "prune index node %u",
01418                          (unsigned)node->nodeid);
01419             hashtb_delete(e);
01420             continue;
01421         }
01422         node->activity /= 2; /* Age the node's activity */
01423         if (node->clean != node->buf->length ||
01424             (node->iodata != NULL && node->activity == 0)) {
01425             if (h->toclean == NULL) {
01426                 h->toclean = ccn_indexbuf_create();
01427                 if (h->toclean == NULL)
01428                     break;
01429             }
01430             ccn_indexbuf_append_element(h->toclean, node->nodeid);
01431         }
01432         hashtb_next(e);
01433     }
01434     hashtb_end(e);
01435     /* If nothing to do, shut down cleaner */
01436     if ((h->toclean == NULL || h->toclean->n == 0) && overquota <= 0 &&
01437         h->btree->io->openfds <= CCN_BT_OPEN_NODES_IDLE) {
01438         h->btree->cleanreq = 0;
01439         h->index_cleaner = NULL;
01440         ccn_indexbuf_destroy(&h->toclean);
01441         if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINE))
01442             ccnr_msg(h, "index btree nodes all clean");
01443         
01444         return(0);
01445     }
01446     return(nrand48(h->seed) % (2U * CCN_BT_CLEAN_TICK_MICROS) + 500);
01447 }
01448 
01449 PUBLIC void
01450 r_store_index_needs_cleaning(struct ccnr_handle *h)
01451 {
01452     int k;
01453     if (h->btree != NULL && h->btree->io != NULL && h->btree->cleanreq > 0) {
01454         if (h->index_cleaner == NULL) {
01455             h->index_cleaner = ccn_schedule_event(h->sched,
01456                                                   CCN_BT_CLEAN_TICK_MICROS,
01457                                                   r_store_index_cleaner, NULL, 0);
01458             if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01459                 ccnr_msg(h, "index cleaner started");
01460         }
01461         /* If necessary, clean in a hurry. */
01462         for (k = 30; /* Backstop to make sure we do not loop here */
01463              k > 0 && h->index_cleaner != NULL &&
01464              h->btree->io->openfds > CCN_BT_OPEN_NODES_LIMIT - 2; k--)
01465             r_store_index_cleaner(h->sched, h, h->index_cleaner, 0);
01466         if (k == 0)
01467             ccnr_msg(h, "index cleaner is in trouble");
01468     }
01469 }
01470 
01471 #undef FAILIF
01472 #undef CHKSYS
01473 #undef CHKRES
01474 #undef CHKPTR

Generated on Thu Feb 16 00:43:58 2012 for Content-Centric Networking in C by  doxygen 1.5.6