// This is how a question will be formatted in JSON. These will be in a dictionary // formatted as {question ID:question data}. var sample_question = { "question": "What is the best explanation for this law?", "answers": ["Option A", "Option B", "Option C", "Option D"], "correct_answer_index": 0, "top10_e5": ["Passage 1 from E5", "Passage 2 from E5", "Passage 3..."], "generation_e5": "This generation is based on the E5 passages.", "top10_colbert": ["Passage 1 from ColBERT", "Passage 2 from Colbert", "..."], "generation_colbert": "This generation is based on the ColBERT passages.", "top10_contains_gold_passage": false, // in either case "gold_passage": "This is the gold passage.", "gold_passage_generation": "This generation answers the question with the gold passage." } // Each user response will take this form. A list of these objects will be uploaded. var user_response = { "user_id": "email@email.com", // track who the respondent is "question_id": "ID ###", // track which question they are responding to "user_answer": 0, // A, B, C, or D "e5_scores": [[1, 2.5, 1.5, 3.5], [0, 1, 4, 1.5]], // [is a law, relevance, quality, helpfulness] "colbert_scores": [[1, 3, 3, 3], [1, 5, 4, 5]], // same format as above } // One JSON file for each user -> check if the file exists, download if it does; otherwise, create. var user_data = { "user_id": "email@email.com", "order": ["question ID 1", "question ID 2", "...", "question ID 15"], // randomized and different for each user "modes": [["e5", "colbert"], ["colbert", "e5"], ["colbert", "e5"], "..."], // randomized and different for each user "current": 3, // index in "order" that the user has not yet completed (starts at 0) -> this user would log in at 3 "responses": [user_response, user_response] // list of user responses in the same order } // When the file exists, download it and make a LOCAL copy. Modify the local copy (e.g. increasing order // and adding responses), and after each question, rewrite the original JSON object with the new data. // To avoid losing data in the case of a bad overwrite, each individual user_response should also be added to // the commit scheduler.