From c00e3afcdc8cf1503439a174621090b133ad329f Mon Sep 17 00:00:00 2001 From: default Date: Fri, 2 Dec 2022 19:14:59 +0100 Subject: [PATCH] Big data storage change. --- activitypub.c | 6 +-- data.c | 77 +++++++++++++++------------ html.c | 142 ++++++++++++++++++++++++++------------------------ main.c | 2 +- snac.h | 12 +++-- 5 files changed, 131 insertions(+), 108 deletions(-) diff --git a/activitypub.c b/activitypub.c index 7f63310..8e173c5 100644 --- a/activitypub.c +++ b/activitypub.c @@ -879,7 +879,7 @@ int process_message(snac *snac, char *msg, char *req) if (xs_type(object) == XSTYPE_DICT) object = xs_dict_get(object, "id"); - timeline_admire(snac, object, actor, 1); + timeline_admire(snac, msg, object, actor, 1); snac_log(snac, xs_fmt("new 'Like' %s %s", actor, object)); do_notify = 1; } @@ -900,7 +900,7 @@ int process_message(snac *snac, char *msg, char *req) xs *who_o = NULL; if (valid_status(actor_request(snac, who, &who_o))) { - timeline_admire(snac, object, actor, 0); + timeline_admire(snac, msg, object, actor, 0); snac_log(snac, xs_fmt("new 'Announce' %s %s", actor, object)); do_notify = 1; } @@ -1094,7 +1094,7 @@ int activitypub_get_handler(d_char *req, char *q_path, else if (strcmp(p_path, "outbox") == 0) { xs *id = xs_fmt("%s/outbox", snac.actor); - xs *elems = local_list(&snac, 20); + xs *elems = timeline_list(&snac, "public", 20); xs *list = xs_list_new(); msg = msg_collection(&snac, id); char *p, *v; diff --git a/data.c b/data.c index 51ee2c7..e9f3d93 100644 --- a/data.c +++ b/data.c @@ -116,6 +116,7 @@ void user_free(snac *snac) xs_free(snac->config); xs_free(snac->key); xs_free(snac->actor); + xs_free(snac->md5); } @@ -154,6 +155,7 @@ int user_open(snac *snac, char *uid) if ((snac->key = xs_json_loads(key_data)) != NULL) { snac->actor = xs_fmt("%s/%s", srv_baseurl, uid); + snac->md5 = xs_md5_hex(snac->actor, strlen(snac->actor)); ret = 1; } else @@ -566,14 +568,31 @@ int object_del_if_unref(const char *id) } +d_char *_object_metadata(const char *id, const char *idxsfx) +/* returns the content of a metadata index */ +{ + xs *fn = _object_fn(id); + fn = xs_replace_i(fn, ".json", idxsfx); + return index_list(fn, XS_ALL); +} + + d_char *object_children(const char *id) /* returns the list of an object's children */ { - xs *fn = _object_fn(id); + return _object_metadata(id, "_c.idx"); +} - fn = xs_replace_i(fn, ".json", "_c.idx"); - return index_list(fn, XS_ALL); +d_char *object_likes(const char *id) +{ + return _object_metadata(id, "_l.idx"); +} + + +d_char *object_announces(const char *id) +{ + return _object_metadata(id, "_a.idx"); } @@ -818,35 +837,6 @@ d_char *timeline_get(snac *snac, char *fn) } -d_char *_timeline_list(snac *snac, char *directory, int max) -/* returns a list of the timeline filenames */ -{ - xs *spec = xs_fmt("%s/%s/" "*.json", snac->basedir, directory); - int c_max; - - /* maximum number of items in the timeline */ - c_max = xs_number_get(xs_dict_get(srv_config, "max_timeline_entries")); - - /* never more timeline entries than the configured maximum */ - if (max > c_max) - max = c_max; - - return xs_glob_n(spec, 0, 1, max); -} - - -d_char *timeline_list(snac *snac, int max) -{ - return _timeline_list(snac, "timeline", max); -} - - -d_char *local_list(snac *snac, int max) -{ - return _timeline_list(snac, "local", max); -} - - d_char *_timeline_new_fn(snac *snac, char *id) /* creates a new filename */ { @@ -1050,7 +1040,7 @@ int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer } -d_char *timeline_top_level(snac *snac, d_char *list) +d_char *timeline_top_level(d_char *list) /* returns the top level md5 entries from this index */ { d_char *tl = xs_list_new(); @@ -1094,7 +1084,26 @@ d_char *timeline_top_level(snac *snac, d_char *list) } -void timeline_admire(snac *snac, char *id, char *admirer, int like) +d_char *timeline_list(snac *snac, const char *idx_name, int max) +/* returns a timeline */ +{ + int c_max; + + /* maximum number of items in the timeline */ + c_max = xs_number_get(xs_dict_get(srv_config, "max_timeline_entries")); + + /* never more timeline entries than the configured maximum */ + if (max > c_max) + max = c_max; + + xs *idx = xs_fmt("%s/%s.idx", snac->basedir, idx_name); + xs *list = index_list_desc(idx, max); + + return timeline_top_level(list); +} + + +void timeline_admire(snac *snac, char *o_msg, char *id, char *admirer, int like) /* updates a timeline entry with a new admiration */ { xs *ofn = _timeline_find_fn(snac, id); diff --git a/html.c b/html.c index 63b54c3..4c1ca79 100644 --- a/html.c +++ b/html.c @@ -386,7 +386,8 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num) { char *id = xs_dict_get(msg, "id"); char *actor = xs_dict_get(msg, "attributedTo"); - char *meta = xs_dict_get(msg, "_snac"); + xs *likes = object_likes(id); + xs *boosts = object_announces(id); xs *s = xs_str_new(NULL); xs *md5 = xs_md5_hex(id, strlen(id)); @@ -407,20 +408,14 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num) s = xs_str_cat(s, s1); } - { - char *l; + if (xs_list_in(likes, snac->md5) == -1) { + /* not already liked; add button */ + s = html_button(s, "like", L("Like")); + } - l = xs_dict_get(meta, "liked_by"); - if (xs_list_in(l, snac->actor) == -1) { - /* not already liked; add button */ - s = html_button(s, "like", L("Like")); - } - - l = xs_dict_get(meta, "announced_by"); - if (strcmp(actor, snac->actor) == 0 || xs_list_in(l, snac->actor) == -1) { - /* not already boosted or us; add button */ - s = html_button(s, "boost", L("Boost")); - } + if (strcmp(actor, snac->actor) == 0 || xs_list_in(boosts, snac->md5) == -1) { + /* not already boosted or us; add button */ + s = html_button(s, "boost", L("Boost")); } if (strcmp(actor, snac->actor) != 0) { @@ -477,23 +472,20 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num) } -d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, int level, int *num) +d_char *html_entry(snac *snac, d_char *os, char *msg, int local, int level, int *num) { char *id = xs_dict_get(msg, "id"); char *type = xs_dict_get(msg, "type"); - char *meta = xs_dict_get(msg, "_snac"); char *actor; int sensitive = 0; char *v; + xs *likes = NULL; + xs *boosts = NULL; /* do not show non-public messages in the public timeline */ if (local && !is_msg_public(snac, msg)) return os; - /* return if already seen */ - if (xs_set_add(seen, id) == 0) - return os; - xs *s = xs_str_new(NULL); /* top wrap */ @@ -522,6 +514,14 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i return xs_str_cat(os, s); } + else + if (strcmp(type, "Note") != 0) { + s = xs_str_cat(s, "
\n"); + + xs *s1 = xs_fmt("

%s

\n", type); + + return xs_str_cat(os, s); + } /* bring the main actor */ if ((actor = xs_dict_get(msg, "attributedTo")) == NULL) @@ -536,14 +536,14 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i /* if this is our post, add the score */ if (xs_startswith(id, snac->actor)) { - int likes = xs_list_len(xs_dict_get(meta, "liked_by")); - int boosts = xs_list_len(xs_dict_get(meta, "announced_by")); + likes = object_likes(id); + boosts = object_announces(id); /* alternate emojis: %d 👍 %d 🔁 */ xs *s1 = xs_fmt( "
%d ★ %d ↺
\n", - likes, boosts); + xs_list_len(likes), xs_list_len(boosts)); s = xs_str_cat(s, s1); } @@ -553,28 +553,46 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i s = xs_str_cat(s, "
\n"); - /* print the origin of the post, if any */ - if (!xs_is_null(p = xs_dict_get(meta, "referrer"))) { + if (boosts == NULL) + boosts = object_announces(id); + + if (xs_list_len(boosts)) { + /* if somebody boosted this, show as origin */ + p = xs_list_get(boosts, 0); xs *actor_r = NULL; - if (valid_status(actor_get(snac, p, &actor_r))) { + if (xs_list_in(boosts, snac->md5) != -1) { + /* we boosted this */ + xs *s1 = xs_fmt( + "
" + "%s %s
", + snac->actor, xs_dict_get(snac->config, "name"), L("boosted") + ); + + s = xs_str_cat(s, s1); + } + else + if (valid_status(object_get_by_md5(p, &actor_r, NULL))) { char *name; if ((name = xs_dict_get(actor_r, "name")) == NULL) name = xs_dict_get(actor_r, "preferredUsername"); - xs *s1 = xs_fmt( - "
" - "%s %s
\n", - xs_dict_get(actor_r, "id"), - name, - L("boosted") - ); + if (!xs_is_null(name)) { + xs *s1 = xs_fmt( + "
" + "%s %s
\n", + xs_dict_get(actor_r, "id"), + name, + L("boosted") + ); - s = xs_str_cat(s, s1); + s = xs_str_cat(s, s1); + } } } - else + +#if 0 if (!xs_is_null((p = xs_dict_get(meta, "parent"))) && *p) { /* this may happen if any of the autor or the parent aren't here */ xs *s1 = xs_fmt( @@ -586,18 +604,6 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i s = xs_str_cat(s, s1); } else - if (!xs_is_null((p = xs_dict_get(meta, "announced_by"))) && - xs_list_in(p, snac->actor) != -1) { - /* we boosted this */ - xs *s1 = xs_fmt( - "
" - "%s %s
", - snac->actor, xs_dict_get(snac->config, "name"), L("boosted") - ); - - s = xs_str_cat(s, s1); - } - else if (!xs_is_null((p = xs_dict_get(meta, "liked_by"))) && xs_list_in(p, snac->actor) != -1) { /* we liked this */ @@ -609,6 +615,7 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i s = xs_str_cat(s, s1); } +#endif } else s = xs_str_cat(s, "
\n"); @@ -717,12 +724,11 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i s = html_entry_controls(snac, s, msg, *num); /** children **/ - - char *children = xs_dict_get(meta, "children"); - int left = xs_list_len(children); + xs *children = object_children(id); + int left = xs_list_len(children); if (left) { - char *id; + char *p, *cmd5; if (level < 4) s = xs_str_cat(s, "
\n"); @@ -732,16 +738,18 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i if (left > 3) s = xs_str_cat(s, "
...\n"); - while (xs_list_iter(&children, &id)) { - xs *chd = timeline_find(snac, id); + p = children; + while (xs_list_iter(&p, &cmd5)) { + xs *chd = NULL; + object_get_by_md5(cmd5, &chd, NULL); if (left == 3) s = xs_str_cat(s, "
\n"); if (chd != NULL) - s = html_entry(snac, s, chd, seen, local, level + 1, num); + s = html_entry(snac, s, chd, local, level + 1, num); else - snac_debug(snac, 2, xs_fmt("cannot read from timeline child %s", id)); + snac_debug(snac, 2, xs_fmt("cannot read from timeline child %s", cmd5)); left--; } @@ -773,13 +781,10 @@ d_char *html_timeline(snac *snac, char *list, int local) /* returns the HTML for the timeline */ { d_char *s = xs_str_new(NULL); - xs_set seen; char *v; double t = ftime(); int num = 0; - xs_set_init(&seen); - s = html_user_header(snac, s, local); if (!local) @@ -789,9 +794,12 @@ d_char *html_timeline(snac *snac, char *list, int local) s = xs_str_cat(s, "
\n"); while (xs_list_iter(&list, &v)) { - xs *msg = timeline_get(snac, v); + xs *msg = NULL; - s = html_entry(snac, s, msg, &seen, local, 0, &num); + if (!valid_status(object_get_by_md5(v, &msg, NULL))) + continue; + + s = html_entry(snac, s, msg, local, 0, &num); } s = xs_str_cat(s, "
\n"); @@ -830,8 +838,6 @@ d_char *html_timeline(snac *snac, char *list, int local) s = xs_str_cat(s, "\n\n"); - xs_set_free(&seen); - return s; } @@ -1007,7 +1013,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char * status = 200; } else { - xs *list = local_list(&snac, XS_ALL); + xs *list = timeline_list(&snac, "public", XS_ALL); *body = html_timeline(&snac, list, 1); *b_size = strlen(*body); @@ -1033,7 +1039,9 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char * else { snac_debug(&snac, 1, xs_fmt("building timeline")); - xs *list = timeline_list(&snac, XS_ALL); + xs *list = timeline_list(&snac, "private", XS_ALL); + + printf("--> %d\n", xs_list_len(list)); *body = html_timeline(&snac, list, 0); *b_size = strlen(*body); @@ -1098,7 +1106,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char * if (strcmp(p_path, ".rss") == 0) { /* public timeline in RSS format */ d_char *rss; - xs *elems = local_list(&snac, 20); + xs *elems = timeline_list(&snac, "public", 20); xs *bio = not_really_markdown(xs_dict_get(snac.config, "bio")); char *p, *v; @@ -1281,13 +1289,13 @@ int html_post_handler(d_char *req, char *q_path, d_char *payload, int p_size, if (strcmp(action, L("Like")) == 0) { xs *msg = msg_admiration(&snac, id, "Like"); post(&snac, msg); - timeline_admire(&snac, id, snac.actor, 1); + timeline_admire(&snac, msg, id, snac.actor, 1); } else if (strcmp(action, L("Boost")) == 0) { xs *msg = msg_admiration(&snac, id, "Announce"); post(&snac, msg); - timeline_admire(&snac, id, snac.actor, 0); + timeline_admire(&snac, msg, id, snac.actor, 0); } else if (strcmp(action, L("MUTE")) == 0) { diff --git a/main.c b/main.c index a109192..8fbdb88 100644 --- a/main.c +++ b/main.c @@ -167,7 +167,7 @@ int main(int argc, char *argv[]) xs *idx = xs_fmt("%s/private.idx", snac.basedir); xs *list = index_list_desc(idx, 256); - xs *tl = timeline_top_level(&snac, list); + xs *tl = timeline_top_level(list); xs *j = xs_json_dumps_pp(tl, 4); printf("%s\n", j); diff --git a/snac.h b/snac.h index b8b5215..88f335a 100644 --- a/snac.h +++ b/snac.h @@ -33,6 +33,7 @@ typedef struct _snac { d_char *config; /* user configuration */ d_char *key; /* keypair */ d_char *actor; /* actor url */ + d_char *md5; /* actor url md5 */ } snac; int user_open(snac *snac, char *uid); @@ -59,9 +60,14 @@ int index_first(const char *fn, char *buf, int size); d_char *index_list(const char *fn, int max); d_char *index_list_desc(const char *fn, int max); +int object_get_by_md5(const char *md5, d_char **obj, const char *type); int object_del(const char *id); int object_del_if_unref(const char *id); +d_char *object_children(const char *id); +d_char *object_likes(const char *id); +d_char *object_announces(const char *id); + int follower_add(snac *snac, const char *actor); int follower_del(snac *snac, const char *actor); int follower_check(snac *snac, const char *actor); @@ -73,11 +79,11 @@ d_char *_timeline_find_fn(snac *snac, char *id); d_char *timeline_find(snac *snac, char *id); int timeline_del(snac *snac, char *id); d_char *timeline_get(snac *snac, char *fn); -d_char *timeline_list(snac *snac, int max); +d_char *timeline_list(snac *snac, const char *idx_name, int max); int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer); -void timeline_admire(snac *snac, char *id, char *admirer, int like); +void timeline_admire(snac *snac, char *o_msg, char *id, char *admirer, int like); -d_char *timeline_top_level(snac *snac, d_char *list); +d_char *timeline_top_level(d_char *list); d_char *local_list(snac *snac, int max);