From 9b3f587974f37dd62b35a84d85f6fdbe82734471 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Tue, 3 May 2011 12:17:00 -0500 Subject: [PATCH 1/3] Changed record keys to use lists instead of atoms. Using atoms will not scale in the large. Also, using atoms as keys forced the API to convert arbitrary end-user input to atoms, adding another potential drain of the finite atom-space available. - ts_api: url paths are now treated and matched as lists, not atoms. Usernames and API calls all use lists now. - ts_json: key items (username, timeline ids) are now expected to be lists, not atoms. --- src/ts_api.erl | 88 ++++++++++++++++++++++++------------------------- src/ts_json.erl | 15 +++++---- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/ts_api.erl b/src/ts_api.erl index 63437c4..df745cd 100644 --- a/src/ts_api.erl +++ b/src/ts_api.erl @@ -34,19 +34,18 @@ out(YArg) -> dispatch_request(YArg, _Session, []) -> make_json_404(YArg, [{see_docs, "/ts_api_doc"}]); dispatch_request(YArg, Session, [H|T]) -> - Param = path_element_to_atom(H), - case {Session, Param} of - {_, login} -> do_login(YArg); - {_, logout} -> do_logout(YArg); + case {Session, H} of + {_, "login"} -> do_login(YArg); + {_, "logout"} -> do_logout(YArg); - {not_logged_in, _} -> make_json_401(YArg); - {session_expired, _} -> make_json_401(YArg, [{error, "session expired"}]); + {"not_logged_in", _} -> make_json_401(YArg); + {"session_expired", _} -> make_json_401(YArg, [{error, "session expired"}]); - {_S, app} -> dispatch_app(YArg, Session, T); - {_S, users} -> dispatch_user(YArg, Session, T); - {_S, timelines} -> dispatch_timeline(YArg, Session, T); - {_S, entries} -> dispatch_entry(YArg, Session, T); + {_S, "app"} -> dispatch_app(YArg, Session, T); + {_S, "users"} -> dispatch_user(YArg, Session, T); + {_S, "timelines"} -> dispatch_timeline(YArg, Session, T); + {_S, "entries"} -> dispatch_entry(YArg, Session, T); {_S, _Other} -> make_json_404(YArg, [{see_docs, "/ts_api_doc/"}]) end. @@ -59,7 +58,7 @@ dispatch_app(YArg, Session, Params) -> {'GET', ["user_summary", UsernameStr]} -> case {Session#ts_api_session.username, - path_element_to_atom(UsernameStr)} of + UsernameStr} of {Username, Username} -> get_user_summary(YArg, Username); _ -> make_json_401(YArg) @@ -75,10 +74,9 @@ dispatch_app(YArg, Session, Params) -> % -------- Dispatch for /user -------- % dispatch_user(YArg, Session, []) -> - dispatch_user(YArg, Session, [atom_to_list(Session#ts_api_session.username)]); + dispatch_user(YArg, Session, [Session#ts_api_session.username]); -dispatch_user(YArg, Session, [H]) -> - Username = path_element_to_atom(H), +dispatch_user(YArg, Session, [Username]) -> HTTPMethod = (YArg#arg.req)#http_request.method, % compare to the logged-in user @@ -97,8 +95,7 @@ dispatch_user(YArg, Session, [H]) -> dispatch_timeline(YArg, _Session, []) -> make_json_404(YArg, [{see_docs, "/ts_api_doc/timelines.html"}]); -dispatch_timeline(YArg, Session, [UrlUsername|_T] = PathElements) -> - Username = path_element_to_atom(UrlUsername), +dispatch_timeline(YArg, Session, [Username|_T] = PathElements) -> case Session#ts_api_session.username of Username -> dispatch_timeline(YArg, PathElements); @@ -106,8 +103,7 @@ dispatch_timeline(YArg, Session, [UrlUsername|_T] = PathElements) -> end. % just username, list timelines -dispatch_timeline(YArg, [UrlUsername]) -> - Username = path_element_to_atom(UrlUsername), +dispatch_timeline(YArg, [Username]) -> HTTPMethod = (YArg#arg.req)#http_request.method, case HTTPMethod of @@ -115,9 +111,7 @@ dispatch_timeline(YArg, [UrlUsername]) -> _Other -> make_json_405(YArg, [{see_docs, "/ts_api_doc/timelines.html"}]) end; -dispatch_timeline(YArg, [UrlUsername, UrlTimelineId]) -> - Username = path_element_to_atom(UrlUsername), - TimelineId = path_element_to_atom(UrlTimelineId), +dispatch_timeline(YArg, [Username, TimelineId]) -> HTTPMethod = (YArg#arg.req)#http_request.method, case HTTPMethod of @@ -136,17 +130,14 @@ dispatch_timeline(YArg, _Other) -> dispatch_entry(YArg, _Session, []) -> make_json_404(YArg, [{see_docs, "/ts_aip_doc/entries.html"}]); -dispatch_entry(YArg, Session, [UrlUsername|_T] = PathElements) -> - Username = path_element_to_atom(UrlUsername), +dispatch_entry(YArg, Session, [Username|_T] = PathElements) -> case Session#ts_api_session.username of Username -> dispatch_entry(YArg, PathElements); _Other -> make_json_404(YArg, [{see_docs, "/ts_api_doc/entries.html"}]) end. -dispatch_entry(YArg, [UrlUsername, UrlTimelineId]) -> - Username = path_element_to_atom(UrlUsername), - TimelineId = path_element_to_atom(UrlTimelineId), +dispatch_entry(YArg, [Username, TimelineId]) -> HTTPMethod = (YArg#arg.req)#http_request.method, case HTTPMethod of @@ -155,9 +146,7 @@ dispatch_entry(YArg, [UrlUsername, UrlTimelineId]) -> _Other -> make_json_405(YArg, [{see_docs, "/ts_api_doc/entries.html"}]) end; -dispatch_entry(YArg, [UrlUsername, UrlTimelineId, UrlEntryId]) -> - Username = path_element_to_atom(UrlUsername), - TimelineId = path_element_to_atom(UrlTimelineId), +dispatch_entry(YArg, [Username, TimelineId, UrlEntryId]) -> EntryId = list_to_integer(UrlEntryId), % TODO: catch non-numbers HTTPMethod = (YArg#arg.req)#http_request.method, @@ -184,8 +173,7 @@ do_login(YArg) -> lists:keyfind(password, 1, Fields)} of % username and password found - {{username, UnameField}, {password, Password}} -> - Username = list_to_atom(UnameField), + {{username, Username}, {password, Password}} -> % check the uname, password case ts_user:check_credentials(Username, Password) of @@ -281,7 +269,11 @@ post_timeline(YArg, Username, TimelineId) -> EJSON = parse_json_body(YArg), % parse into a Timeline record - TR = ts_json:ejson_to_record(#ts_timeline{}, EJSON), + TR = try ts_json:ejson_to_record(#ts_timeline{}, EJSON) + catch _:InputError -> + error_logger:error_report("Bad input: ~p", [InputError]), + throw(make_json_400(YArg)) + end, % set username and timeline id NewRecord = TR#ts_timeline{ref = {Username, TimelineId}}, @@ -309,12 +301,16 @@ put_timeline(YArg, Username, TimelineId) -> %{struct, Fields} = EJSON, % parse into a timeline record - TR = ts_json:ejson_to_record(#ts_timeline{}, EJSON), + TR = try ts_json:ejson_to_record(#ts_timeline{}, EJSON) + catch _:InputError -> + error_logger:error_report("Bad input: ~p", [InputError]), + throw(make_json_400(YArg)) + end, % not supported right now, would require changing all the entry keys % check to see if they are changing the timeline id %NewTimelineId = case lists:keyfind(1, timeline_id, Fields) of - % {timeline_id, Field} -> list_to_atom(Field); + % {timeline_id, Field} -> Field; % false -> TimelineId end, % set username and timeline id @@ -428,7 +424,11 @@ post_entry(YArg, Username, TimelineId) -> EJSON = parse_json_body(YArg), % parse into ts_entry record - ER = ts_json:ejson_to_record(#ts_entry{}, EJSON), + ER = try ts_json:ejson_to_record(#ts_entry{}, EJSON) + catch _:InputError -> + error_logger:error_report("Bad input: ~p", [InputError]), + throw(make_json_400(YArg)) + end, % set username and timeline id NewRecord = ER#ts_entry{ref = {Username, TimelineId, undef}}, @@ -455,7 +455,11 @@ put_entry(YArg, Username, TimelineId, EntryId) -> EJSON = parse_json_body(YArg), % parse into ts_entry record - ER = ts_json:ejson_to_record(#ts_entry{}, EJSON), + ER = try ts_json:ejson_to_record(#ts_entry{}, EJSON) + catch _:InputError -> + error_logger:error_report("Bad input: ~p", [InputError]), + throw(make_json_400(YArg)) + end, % set uername, timeline id, and entry id NewRecord = ER#ts_entry{ref = {Username, TimelineId, EntryId}}, @@ -492,10 +496,6 @@ delete_entry(YArg, Username, TimelineId, EntryId) -> % ======== UTIL METHODS ======== % % ============================== % -%% Convert one path element to an atom. -path_element_to_atom(PE) -> - list_to_atom(re:replace(PE, "\\s", "_", [{return, list}])). - parse_json_body(YArg) -> case catch json:decode([], binary_to_list(YArg#arg.clidata)) of {done, {ok, EJSON}, _} -> EJSON; @@ -547,19 +547,19 @@ make_json_405(YArg) -> make_json_405(YArg, []). make_json_405(YArg, Fields) -> % add default status if not provided F1 = case lists:keyfind(status, 1, Fields) of - false -> Fields ++ [{status, method_not_allowed}]; + false -> Fields ++ [{status, "method not allowed"}]; _Else -> Fields end, % add the path they requested - F2 = F1 ++ [{path, (YArg#arg.req)#http_request.path}], + % F2 = F1 ++ [{path, io_lib:format("~s", [(YArg#arg.req)#http_request.path])}], - [{status, 405}, {content, "application/json", json:encode({struct, F2})}]. + [{status, 405}, {content, "application/json", json:encode({struct, F1})}]. make_json_500(_YArg, Error) -> EJSON = {struct, [ {status, "internal server error"}, - {error, io_lib:format("~p", [Error])}]}, + {error, io_lib:format("~s", [Error])}]}, [{status, 500}, {content, "application/json", json:encode(EJSON)}]. make_json_500(_YArg) -> diff --git a/src/ts_json.erl b/src/ts_json.erl index 0f10a79..4079353 100644 --- a/src/ts_json.erl +++ b/src/ts_json.erl @@ -7,10 +7,11 @@ encode_record(Record) -> lists:flatten(json:encode(record_to_ejson(Record))). record_to_ejson(Record=#ts_user{}) -> {struct, [ - {id, atom_to_list(Record#ts_user.username)}, + {id, Record#ts_user.username}, {name, Record#ts_user.name}, {email, Record#ts_user.email}, - {join_date, encode_datetime(Record#ts_user.join_date)}]}; + {join_date, encode_datetime(Record#ts_user.join_date)}, + {ext_data, Record#ts_user.ext_data}]}; record_to_ejson(Record=#ts_timeline{}) -> % pull out the username and timeline id @@ -18,8 +19,8 @@ record_to_ejson(Record=#ts_timeline{}) -> % create the EJSON struct {struct, [ - {user_id, atom_to_list(Username)}, - {id, atom_to_list(TimelineId)}, + {user_id, Username}, + {id, TimelineId}, {created, encode_datetime(Record#ts_timeline.created)}, {description, Record#ts_timeline.desc}]}; @@ -32,15 +33,15 @@ record_to_ejson(Record=#ts_entry{}) -> % create the EJSON struct {struct, [ - {user_id, atom_to_list(Username)}, - {timeline_id, atom_to_list(TimelineId)}, + {user_id, Username}, + {timeline_id, TimelineId}, {id, EntryId}, {timestamp, encode_datetime(DateTime)}, {mark, Record#ts_entry.mark}, {notes, Record#ts_entry.notes}]}. encode_datetime({{Year, Month, Day}, {Hour, Minute, Second}}) -> - lists:flatten(io_lib:format("~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B~3.10.0BZ", + lists:flatten(io_lib:format("~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.~3.10.0BZ", [Year, Month, Day, Hour, Minute, Second, 000])). ejson_to_record(_Empty=#ts_timeline{}, EJSON) -> From ebd4b117c92bfdba2bb65654ca6210ee575cdd1b Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Tue, 3 May 2011 12:09:28 -0500 Subject: [PATCH 2/3] Updated database to use lists instead of atoms for table keys. --- db/test/id_counter.DCL | Bin 0 -> 169 bytes db/test/schema.DAT | Bin 8278 -> 8323 bytes db/test/ts_entry.DCD | Bin 363 -> 196 bytes db/test/ts_timeline.DCD | Bin 201 -> 190 bytes db/test/ts_timeline.DCL | Bin 0 -> 237 bytes db/test/ts_user.DCD | Bin 254 -> 258 bytes 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 db/test/id_counter.DCL create mode 100644 db/test/ts_timeline.DCL diff --git a/db/test/id_counter.DCL b/db/test/id_counter.DCL new file mode 100644 index 0000000000000000000000000000000000000000..de6df2681fa2554ae1ffec95bb5c5d58fa46a2bf GIT binary patch literal 169 zcmZQ%VrEH>@a1M;VDL%`_i=2_U`t`(%E?cU&qz&7Ni9lYU{6WTi3bW~GcX(K8DulC zndq7585*T9@aE;`<)@@N{8w1^WO19XPW=0TKeU;uKj33HuT>R(Jpa0rRs&9UB1w z2a{kPHw$C{2xWM5Uu0o)VUw30Y9<3=0B8ba01a+$Zf|5|KyGhnZ*z2L17ZLG1sGxg S1W^ZK00H)fvjrbe0Re;fkRKcX delta 69 zcmV-L0J{H!LDoP70018bu>{8w18j?EdYDGVGX#qp_mB}J7P%-IZ_St&`W zMR|!uDcKAx<@rU~iK=<53^Rg_j`L+R7>1-4mt^LpE99k?E2N|rXQt=rDLCd7=PQ&z VB~nu|OMpUoDGG_{iJ5tNSpa#eC?Ws= literal 363 zcma*jO-chX7{>8eXQqBpHzGw>VRWGg!Iq-XjfyC3vE9so-3^(^+h&?+Qj5E|+Qx_uVEKRg?oL0F6idjQbSpP%kNM2*94r6-v{#?f%t;swwAAR%bdO9{0$ zZPR9pLtSa5Io>KQv~?oL0iuH(>R)d;gufSF9RG_EhizvwRL(3#z+q>~b2N$>&LLcA zBU!8>XtaL5_az+l(k@o>4tY=xO{pftm4!rSsf6R9cElj7swE};SL?`F8)AiCP+xmo zw|3%dpI58eVOq{Gacnk6(;+EZ=ZkdewZXdDsn=evS6YKrXxv%cBlk>^E_)}VB)N&t J{dGf4{{V=oZKePK diff --git a/db/test/ts_timeline.DCD b/db/test/ts_timeline.DCD index c532746b67098ef26e969e5af975d071fe7b47f7..1cef15a59a3785bdc8bfc476d41f3c087b78971d 100644 GIT binary patch delta 122 zcmX@fxQ}syg_=y!18-h_UVcidLtcJHesM_#a}omst3VP1`-ay^3@ka985kJSC&r0~ zq%d%1r6i>m$WX@*L49UnWRsfhd5WiG%1 zv`>HiUMCT0_473Xuu;41`d9Y|!`kml&5}3}`c&0rJ6MQ$F|+6`i)Ari5I&rDo_OWF z_D$460o{p0N^6C|?cR#lmnd{X7{#WH=E${itP_jDffNTP16iaOAFNK_iIV3}|4(-0 lR0}zlpE9$^zFRqOPV8dtUH;&5Sw(X;(Qs1`x|d)g{{xa6Md1Jd literal 0 HcmV?d00001 diff --git a/db/test/ts_user.DCD b/db/test/ts_user.DCD index 1e2c37608886a0f87cd27faaa6b4d68aa64d2a26..05cef8295eba6515ed9ce6547b1a8ba8a91733ab 100644 GIT binary patch delta 153 zcmeyz*u=!c$i&Q&9O28&z`)=&k;l1SB8h<|k}-*ai+>qVXhBlAk7IKNM+yUbNpXB> zacWUE17}uBQfg6NVo^#iP=(0Fgn3TF*5 z%KLaDyUl&;>K~O^At&Ev|M(axyLD;r&V-&v+p-zNyz=uBOEMDk6kHO^GE*j=HfGEM E0Imc%Q2+n{ delta 131 zcmZo-`p3w_$i&Q&9O28&z`)=>k;ge(FmY=V17Gt(Ab);RxQ}CV273wvdr5J8X>n>% z3Ik_WN>XZ3USd&7E>MNYRj#=3U;S3*jkc?T%Kxk~e!RwT14u%}M6R7d--AKRVP9Xf g=^MXa%R&_65Bq&&-Jks4?z2z~OM#B(i-~((02GWY`Tzg` From 3d89477f7e1095b40478d57433dadf3daada8e63 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Tue, 3 May 2011 12:18:20 -0500 Subject: [PATCH 3/3] Added ext_data field to ts_user. --- src/ts_db_records.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ts_db_records.hrl b/src/ts_db_records.hrl index 2cfc828..2d94a7b 100644 --- a/src/ts_db_records.hrl +++ b/src/ts_db_records.hrl @@ -4,8 +4,8 @@ pwd_salt, name, email, - join_date%, -% ext_data % other extensible data + join_date, + ext_data = [] % other extensible data }). % ts_user.ext_data can be: