00001 /** 00002 * @file ccn/btree.h 00003 * BTree 00004 */ 00005 /* (Will be) Part of the CCNx C Library. 00006 * 00007 * Copyright (C) 2011 Palo Alto Research Center, Inc. 00008 * 00009 * This library is free software; you can redistribute it and/or modify it 00010 * under the terms of the GNU Lesser General Public License version 2.1 00011 * as published by the Free Software Foundation. 00012 * This library is distributed in the hope that it will be useful, 00013 * but WITHOUT ANY WARRANTY; without even the implied warranty of 00014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00015 * Lesser General Public License for more details. You should have received 00016 * a copy of the GNU Lesser General Public License along with this library; 00017 * if not, write to the Free Software Foundation, Inc., 51 Franklin Street, 00018 * Fifth Floor, Boston, MA 02110-1301 USA. 00019 */ 00020 00021 #ifndef CCN_BTREE_DEFINED 00022 #define CCN_BTREE_DEFINED 00023 00024 #include <stdio.h> 00025 #include <sys/types.h> 00026 #include <ccn/charbuf.h> 00027 #include <ccn/hashtb.h> 00028 00029 struct ccn_btree_io; 00030 struct ccn_btree_node; 00031 00032 /** 00033 * Methods for external I/O of btree nodes. 00034 * 00035 * These are supplied by the client, and provide an abstraction 00036 * to hold the persistent representation of the btree. 00037 * 00038 * Each node has a nodeid that serves as its filename. These start as 1 and 00039 * are assigned consecutively. The node may correspond to a file in a file 00040 * system, or to some other abstraction as appropriate. 00041 * 00042 * Open should prepare for I/O to a node. It may use the iodata slot to 00043 * keep track of its state, and should set iodata to a non-NULL value. 00044 * It should update the count of openfds as appropriate. 00045 * 00046 * Read gets bytes from the file and places it into the buffer at the 00047 * corresponding position. The parameter is a limit for the max buffer size. 00048 * Bytes prior to the clean mark do not need to be read. 00049 * The buffer should be extended, if necessary, to hold the data. 00050 * Read is not responsible for updating the clean mark. 00051 * 00052 * Write puts bytes from the buffer into the file, and truncates the file 00053 * according to the buffer length. Bytes prior to the clean mork do not 00054 * need to be written, since they should be the same in the buffer and the 00055 * file. Write is not responsible for updating the clean mark. 00056 * 00057 * Close is called at the obvious time. It should free any node io state and 00058 * set iodata to NULL, updating openfds as appropriate. It should not change 00059 * the other parts of the node. 00060 * 00061 * Negative return values indicate errors. 00062 */ 00063 typedef int (*ccn_btree_io_openfn) 00064 (struct ccn_btree_io *, struct ccn_btree_node *); 00065 typedef int (*ccn_btree_io_readfn) 00066 (struct ccn_btree_io *, struct ccn_btree_node *, unsigned); 00067 typedef int (*ccn_btree_io_writefn) 00068 (struct ccn_btree_io *, struct ccn_btree_node *); 00069 typedef int (*ccn_btree_io_closefn) 00070 (struct ccn_btree_io *, struct ccn_btree_node *); 00071 typedef int (*ccn_btree_io_destroyfn) 00072 (struct ccn_btree_io **); 00073 00074 /* This serves as the external name of a btree node. */ 00075 typedef unsigned ccn_btnodeid; 00076 00077 /** 00078 * Holds the methods and the associated common data. 00079 */ 00080 struct ccn_btree_io { 00081 char clue[16]; /* unused except for debugging/logging */ 00082 ccn_btree_io_openfn btopen; 00083 ccn_btree_io_readfn btread; 00084 ccn_btree_io_writefn btwrite; 00085 ccn_btree_io_closefn btclose; 00086 ccn_btree_io_destroyfn btdestroy; 00087 ccn_btnodeid maxnodeid; /**< Largest assigned nodeid */ 00088 int openfds; /**< Number of open files */ 00089 void *data; 00090 }; 00091 /** 00092 * State associated with a btree node 00093 * 00094 * These usually live in the resident hashtb of a ccn_btree, but might be 00095 * elsewhere (such as stack-allocated) in some cases. 00096 */ 00097 struct ccn_btree_node { 00098 ccn_btnodeid nodeid; /**< Identity of node */ 00099 struct ccn_charbuf *buf; /**< The internal buffer */ 00100 void *iodata; /**< Private use by ccn_btree_io methods */ 00101 ccn_btnodeid parent; /**< Parent node id; 0 if unknown */ 00102 unsigned clean; /**< Number of stable buffered bytes at front */ 00103 unsigned freelow; /**< Index of first unused byte of free space */ 00104 unsigned corrupt; /**< Structure is not to be trusted */ 00105 unsigned activity; /**< Meters use of the node */ 00106 }; 00107 00108 /** Increment to node->activity when node is referenced but not changed */ 00109 #define CCN_BT_ACTIVITY_REFERENCE_BUMP 1 00110 /** Increment to node->activity when node is read from disk */ 00111 #define CCN_BT_ACTIVITY_READ_BUMP 8 00112 /** Increment to node->activity when node is modified */ 00113 #define CCN_BT_ACTIVITY_UPDATE_BUMP 16 00114 00115 /** Limit to the number of btree nodes kept open when idle */ 00116 #define CCN_BT_OPEN_NODES_IDLE 5 00117 /** Limit to the number of file descriptors the btree should use at a time */ 00118 #define CCN_BT_OPEN_NODES_LIMIT 13 00119 00120 00121 /** 00122 * State associated with a btree as a whole 00123 */ 00124 struct ccn_btree { 00125 unsigned magic; /**< for making sure we point to a btree */ 00126 ccn_btnodeid nextnodeid; /**< for allocating new btree nodes */ 00127 struct ccn_btree_io *io; /**< storage layer */ 00128 struct hashtb *resident; /**< of ccn_btree_node, by nodeid */ 00129 ccn_btnodeid nextsplit; /**< oversize node that needs splitting */ 00130 ccn_btnodeid missedsplit; /**< should stay zero */ 00131 int errors; /**< counter for detected errors */ 00132 int cleanreq; /**< if nonzero, cleaning might be needed */ 00133 /* tunables */ 00134 int full; /**< split internal nodes bigger than this */ 00135 int full0; /**< split leaf nodes bigger than this */ 00136 int nodebytes; /**< limit size of node */ 00137 int nodepool; /**< limit resident size */ 00138 }; 00139 00140 /** 00141 * Structure of a node. 00142 * 00143 * These are as they appear on external storage, so we stick to 00144 * single-byte types to keep it portable between machines. 00145 * Multi-byte numeric fields are always in big-endian format. 00146 * 00147 * Within a node, the entries are fixed size. 00148 * The entries are packed together at the end of the node's storage, 00149 * so that by examining the last entry the location of the other entries 00150 * can be determined directly. The entsz field includes the whole entry, 00151 * which consists of a payload followed by a trailer. 00152 * 00153 * The keys are stored in the first portion of the node. They may be 00154 * in multiple pieces, and the pieces may overlap arbitrarily. This offers 00155 * a very simple form of compression, since the keys within a node are 00156 * very likely to have a lot in common with each other. 00157 * 00158 * A few bytes at the very beginning serve as a header. 00159 * 00160 * This is the overall structure of a node: 00161 * 00162 * +---+-----------------------+--------------+----+----+-- --+----+ 00163 * |hdr|..string......space....| (free space) | E0 | E1 | ... | En | 00164 * +---+-----------------------+--------------+----+----+-- --+----+ 00165 * 00166 * It is designed so that new entries can be added without having to 00167 * rewrite all of the string space. Thus the header should not contain 00168 * things that we expect to change often. 00169 */ 00170 struct ccn_btree_node_header { 00171 unsigned char magic[4]; /**< File magic */ 00172 unsigned char version[1]; /**< Format version */ 00173 unsigned char nodetype[1]; /**< Indicates root node, backup root, etc. */ 00174 unsigned char level[1]; /**< Level within the tree */ 00175 unsigned char extsz[1]; /**< Header extension size (CCN_BT_SIZE_UNITS)*/ 00176 }; 00177 00178 /** 00179 * Structure of a node entry trailer. 00180 * 00181 * This is how the last few bytes of each entry within a node are arranged. 00182 * 00183 */ 00184 struct ccn_btree_entry_trailer { 00185 unsigned char koff0[4]; /**< offset of piece 0 of the key */ 00186 unsigned char ksiz0[2]; /**< size of piece 0 of the key */ 00187 unsigned char koff1[4]; /**< offset of piece 1 */ 00188 unsigned char ksiz1[2]; /**< size of piece 1 */ 00189 unsigned char entdx[2]; /**< index of this entry within the node */ 00190 unsigned char level[1]; /**< leaf nodes are at level 0 */ 00191 unsigned char entsz[1]; /**< entry size in CCN_BT_SIZE_UNITS */ 00192 }; 00193 #define CCN_BT_SIZE_UNITS 8 00194 /** Maximum key size, dictated by size of above size fields */ 00195 #define CCN_BT_MAX_KEY_SIZE 65535 00196 00197 /** 00198 * Structure of the entry payload within an internal (non-leaf) node. 00199 */ 00200 struct ccn_btree_internal_payload { 00201 unsigned char magic[1]; /**< CCN_BT_INTERNAL_MAGIC */ 00202 unsigned char pad[3]; /**< must be zero */ 00203 unsigned char child[4]; /**< nodeid of a child */ 00204 }; 00205 #define CCN_BT_INTERNAL_MAGIC 0xCC 00206 /** 00207 * Logical structure of the entry within an internal (non-leaf) node. 00208 */ 00209 struct ccn_btree_internal_entry { 00210 struct ccn_btree_internal_payload ie; 00211 struct ccn_btree_entry_trailer trailer; 00212 }; 00213 00214 /* More extensive function descriptions are provided in the code. */ 00215 00216 /* Number of entries within the node */ 00217 int ccn_btree_node_nent(struct ccn_btree_node *node); 00218 00219 /* Node level (leaves are at level 0) */ 00220 int ccn_btree_node_level(struct ccn_btree_node *node); 00221 00222 /* Node entry size */ 00223 int ccn_btree_node_getentrysize(struct ccn_btree_node *node); 00224 00225 /* Node payload size */ 00226 int ccn_btree_node_payloadsize(struct ccn_btree_node *node); 00227 00228 /* Get address of the indexed entry within node */ 00229 void *ccn_btree_node_getentry(size_t payload_bytes, 00230 struct ccn_btree_node *node, int i); 00231 00232 /* Fetch the indexed key and place it into dst */ 00233 int ccn_btree_key_fetch(struct ccn_charbuf *dst, 00234 struct ccn_btree_node *node, int i); 00235 00236 /* Append the indexed key to dst */ 00237 int ccn_btree_key_append(struct ccn_charbuf *dst, 00238 struct ccn_btree_node *node, int i); 00239 00240 /* Compare given key with the key in the indexed entry of the node */ 00241 int ccn_btree_compare(const unsigned char *key, size_t size, 00242 struct ccn_btree_node *node, int i); 00243 00244 #define CCN_BT_ENCRES(ndx, success) (2 * (ndx) + ((success) || 0)) 00245 #define CCN_BT_SRCH_FOUND(res) ((res) & 1) 00246 #define CCN_BT_SRCH_INDEX(res) ((res) >> 1) 00247 /* Search within the node for the key, or something near it */ 00248 int ccn_btree_searchnode(const unsigned char *key, size_t size, 00249 struct ccn_btree_node *node); 00250 00251 /* Insert a new entry at slot i of node */ 00252 int ccn_btree_insert_entry(struct ccn_btree_node *node, int i, 00253 const unsigned char *key, size_t keysize, 00254 void *payload, size_t payload_bytes); 00255 00256 /* Initialize a btree node */ 00257 int ccn_btree_init_node(struct ccn_btree_node *node, 00258 int level, unsigned char nodetype, unsigned char extsz); 00259 00260 /* Test for an oversize node */ 00261 int ccn_btree_oversize(struct ccn_btree *btree, struct ccn_btree_node *node); 00262 00263 /* Check a node for internal consistency */ 00264 int ccn_btree_chknode(struct ccn_btree_node *node); 00265 00266 /* 00267 * Overall btree operations 00268 */ 00269 00270 /* Handle creation and destruction */ 00271 struct ccn_btree *ccn_btree_create(void); 00272 int ccn_btree_destroy(struct ccn_btree **); 00273 00274 /* Record an error */ 00275 void ccn_btree_note_error(struct ccn_btree *bt, int info); 00276 00277 /* Access a node, creating or reading it if necessary */ 00278 struct ccn_btree_node *ccn_btree_getnode(struct ccn_btree *bt, 00279 ccn_btnodeid nodeid, 00280 ccn_btnodeid parentid); 00281 00282 /* Get a node handle if it is already resident */ 00283 struct ccn_btree_node *ccn_btree_rnode(struct ccn_btree *bt, 00284 ccn_btnodeid nodeid); 00285 00286 /* Clean a node and release io resources, retaining cached node in memory */ 00287 int ccn_btree_close_node(struct ccn_btree *btree, struct ccn_btree_node *node); 00288 00289 /* Do a lookup, starting from the default root */ 00290 int ccn_btree_lookup(struct ccn_btree *btree, 00291 const unsigned char *key, size_t size, 00292 struct ccn_btree_node **leafp); 00293 00294 /* Do a lookup, starting from the provided root and stopping at stoplevel */ 00295 int ccn_btree_lookup_internal(struct ccn_btree *btree, 00296 struct ccn_btree_node *root, int stoplevel, 00297 const unsigned char *key, size_t size, 00298 struct ccn_btree_node **ansp); 00299 00300 /* Find the leaf that comes after the given node */ 00301 int ccn_btree_next_leaf(struct ccn_btree *btree, 00302 struct ccn_btree_node *node, 00303 struct ccn_btree_node **ansp); 00304 00305 /* Find the leaf that comes before the given node */ 00306 int ccn_btree_prev_leaf(struct ccn_btree *btree, 00307 struct ccn_btree_node *node, 00308 struct ccn_btree_node **ansp); 00309 00310 /* Split a node into two */ 00311 int ccn_btree_split(struct ccn_btree *btree, struct ccn_btree_node *node); 00312 00313 /* Prepare to update a node */ 00314 int ccn_btree_prepare_for_update(struct ccn_btree *bt, 00315 struct ccn_btree_node *node); 00316 00317 /* Check the whole btree carefully */ 00318 int ccn_btree_check(struct ccn_btree *btree, FILE *outfp); 00319 00320 /* 00321 * Storage layer - client can provide other options 00322 */ 00323 00324 /* For btree node storage in files */ 00325 struct ccn_btree_io *ccn_btree_io_from_directory(const char *path, 00326 struct ccn_charbuf *msgs); 00327 00328 /* Low-level field access */ 00329 unsigned ccn_btree_fetchval(const unsigned char *p, int size); 00330 void ccn_btree_storeval(unsigned char *p, int size, unsigned v); 00331 00332 #endif