From 4f3ecda14c16edbd8cbf532d8ab6c123ccbb2160 Mon Sep 17 00:00:00 2001 From: andrea Date: Fri, 22 Sep 2023 10:29:10 +0200 Subject: [PATCH] Add metadata headers --- client/client.go | 1 + go.mod | 1 + go.sum | 2 + models/meta.go | 8 + models/quiz.go | 3 +- store/file/file.go | 269 +++++++++++++++++++++++++- store/file/file_test.go | 55 +++++- store/file/testdata/quizzes/quiz_1.md | 4 + store/file/testdata/quizzes/quiz_2.md | 4 + store/file/testdata/quizzes/quiz_3.md | 4 + store/file/testdata/quizzes/quiz_4.md | 10 + store/file/testdata/quizzes/quiz_5.md | 7 + store/memory/memory.go | 11 +- 13 files changed, 358 insertions(+), 21 deletions(-) create mode 100644 models/meta.go create mode 100644 store/file/testdata/quizzes/quiz_4.md create mode 100644 store/file/testdata/quizzes/quiz_5.md diff --git a/client/client.go b/client/client.go index f7b5dfd..d06cd14 100644 --- a/client/client.go +++ b/client/client.go @@ -46,6 +46,7 @@ type CreateAnswerRequest struct { type CreateUpdateQuizRequest struct { *Quiz + *models.Meta } type DeleteQuizRequest struct { diff --git a/go.mod b/go.mod index 54b7830..3c67a7b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require github.com/sirupsen/logrus v1.8.1 require ( + github.com/go-yaml/yaml v2.1.0+incompatible // indirect github.com/google/uuid v1.3.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kr/pretty v0.2.1 // indirect diff --git a/go.sum b/go.sum index 1796bf2..08d7fd2 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= diff --git a/models/meta.go b/models/meta.go new file mode 100644 index 0000000..0f8a44e --- /dev/null +++ b/models/meta.go @@ -0,0 +1,8 @@ +package models + +import "time" + +type Meta struct { + ID string `json:"id" yaml:"id"` + CreatedAt time.Time `json:"created_at" yaml:"created_at"` +} diff --git a/models/quiz.go b/models/quiz.go index 8d1ed23..8919ad2 100644 --- a/models/quiz.go +++ b/models/quiz.go @@ -1,7 +1,8 @@ package models type Quiz struct { - ID string `json:"id"` + Meta + // ID string `json:"id"` Hash string `json:"hash"` Question *Question `json:"question"` Answers []*Answer `json:"answers"` diff --git a/store/file/file.go b/store/file/file.go index 4cfcea9..6cf6b18 100644 --- a/store/file/file.go +++ b/store/file/file.go @@ -1,11 +1,15 @@ package file import ( + "bufio" + "bytes" "errors" "fmt" + "io" "io/fs" "io/ioutil" "os" + "path" "path/filepath" "strings" "sync" @@ -15,8 +19,11 @@ import ( "git.andreafazzi.eu/andrea/probo/hasher/sha256" "git.andreafazzi.eu/andrea/probo/models" "git.andreafazzi.eu/andrea/probo/store/memory" + "github.com/go-yaml/yaml" ) +var ErrorMetaHeaderIsNotPresent = errors.New("Meta header was not found in file.") + type FileProboCollectorStore struct { Dir string @@ -73,17 +80,25 @@ func (s *FileProboCollectorStore) Reindex() error { if err != nil { return err } - quiz, err := QuizFromMarkdown(string(content)) + quiz, meta, err := QuizFromMarkdown(string(content)) if err != nil { return err } + q, err := s.memoryStore.CreateQuiz(&client.CreateUpdateQuizRequest{ Quiz: quiz, + Meta: meta, }) if err != nil { return err } + if meta == nil { + s.WriteMetaHeaderToFile(filename, &models.Meta{ + ID: q.ID, + CreatedAt: time.Now(), + }) + } s.SetPath(q, fullPath) } @@ -202,8 +217,13 @@ func MarkdownFromQuiz(quiz *models.Quiz) (string, error) { return markdown, nil } -func QuizFromMarkdown(markdown string) (*client.Quiz, error) { - lines := strings.Split(markdown, "\n") +func QuizFromMarkdown(markdown string) (*client.Quiz, *models.Meta, error) { + meta, remainingMarkdown, err := parseMetaHeaderFromMarkdown(markdown) + if err != nil { + return nil, nil, err + } + + lines := strings.Split(remainingMarkdown, "\n") questionText := "" answers := []*client.Answer{} @@ -225,17 +245,113 @@ func QuizFromMarkdown(markdown string) (*client.Quiz, error) { questionText = strings.TrimRight(questionText, "\n") if questionText == "" { - return nil, fmt.Errorf("Question text should not be empty.") + return nil, nil, fmt.Errorf("Question text should not be empty.") } if len(answers) < 2 { - return nil, fmt.Errorf("Number of answers should be at least 2 but parsed answers are %d.", len(answers)) + return nil, nil, fmt.Errorf("Number of answers should be at least 2 but parsed answers are %d.", len(answers)) } question := &client.Question{Text: questionText} quiz := &client.Quiz{Question: question, Answers: answers} - return quiz, nil + return quiz, meta, nil +} + +func (s *FileProboCollectorStore) ReadMetaHeaderFromFile(filename string) (*models.Meta, error) { + data, err := ioutil.ReadFile(path.Join(s.Dir, filename)) + if err != nil { + return nil, err + } + meta, _, err := parseMetaHeaderFromMarkdown(string(data)) + if err != nil { + return nil, err + } + return meta, nil +} + +func (s *FileProboCollectorStore) WriteMetaHeaderToFile(filename string, meta *models.Meta) (*models.Meta, error) { + readMeta, err := s.ReadMetaHeaderFromFile(filename) + if err != nil { + return nil, err + } + + if readMeta == nil { + _, err := writeMetaHeader(path.Join(s.Dir, filename), meta) + if err != nil { + return nil, err + } + } + + return meta, nil +} + +func (s *FileProboCollectorStore) removeMetaFromFile(filename string) (*models.Meta, error) { + file, err := os.Open(path.Join(s.Dir, filename)) + if err != nil { + return nil, err + } + defer file.Close() + + var buffer bytes.Buffer + + reader := bufio.NewReader(file) + + var meta models.Meta + var line string + var sb strings.Builder + for { + line, err = reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + if strings.TrimSpace(line) == "---" { + break + } + } + + for { + line, err = reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + if strings.TrimSpace(line) == "---" { + break + } + + sb.WriteString(line) + } + + err = yaml.Unmarshal([]byte(sb.String()), &meta) + if err != nil { + return nil, err + } + + _, err = io.Copy(&buffer, reader) + if err != nil { + return nil, err + } + + file, err = os.Create(path.Join(s.Dir, filename)) + if err != nil { + return nil, err + } + defer file.Close() + + _, err = io.Copy(file, &buffer) + if err != nil { + return nil, err + } + + return &meta, nil } func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz) error { @@ -253,10 +369,14 @@ func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz) if err != nil { return err } - defer file.Close() - _, err = file.Write([]byte(markdown)) + markdownWithMetaHeader, err := addMetaHeaderToMarkdown(markdown, &quiz.Meta) + if err != nil { + return err + } + + _, err = file.Write([]byte(markdownWithMetaHeader)) if err != nil { return err } @@ -265,3 +385,136 @@ func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz) return nil } + +func addMetaHeaderToMarkdown(content string, meta *models.Meta) (string, error) { + var buffer bytes.Buffer + + header, err := yaml.Marshal(meta) + if err != nil { + return "", err + } + _, err = buffer.WriteString("---\n" + string(header) + "---\n") + if err != nil { + return "", err + } + + _, err = buffer.WriteString(content) + if err != nil { + return "", err + } + + return buffer.String(), nil +} + +func parseMetaHeaderFromMarkdown(markdown string) (*models.Meta, string, error) { + reader := strings.NewReader(markdown) + var sb strings.Builder + var line string + var err error + for { + line, err = readLine(reader) + if err != nil { + if err == io.EOF { + break + } + return nil, "", err + } + + if strings.TrimSpace(line) == "---" { + break + } + } + + for { + line, err = readLine(reader) + if err != nil { + if err == io.EOF { + break + } + return nil, "", err + } + + if strings.TrimSpace(line) == "---" { + break + } + + sb.WriteString(line) + } + + if sb.String() == "" { + return nil, markdown, nil + } + + var meta models.Meta + err = yaml.Unmarshal([]byte(sb.String()), &meta) + if err != nil { + return nil, markdown, err + } + + remainingMarkdown := markdown[strings.Index(markdown, "---\n"+sb.String()+"---\n")+len("---\n"+sb.String()+"---\n"):] + + return &meta, remainingMarkdown, nil +} + +func readLine(reader *strings.Reader) (string, error) { + var sb strings.Builder + for { + r, _, err := reader.ReadRune() + if err != nil { + if err == io.EOF { + return sb.String(), io.EOF + } + return "", err + } + + sb.WriteRune(r) + if r == '\n' { + break + } + } + + return sb.String(), nil +} + +func writeMetaHeader(filename string, meta *models.Meta) (*models.Meta, error) { + // Apri il file in lettura + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + // Crea un buffer in memoria + var buffer bytes.Buffer + + // Scrivi l'intestazione YAML nel buffer + header, err := yaml.Marshal(meta) + if err != nil { + return nil, err + } + _, err = buffer.WriteString("---\n" + string(header) + "---\n") + if err != nil { + return nil, err + } + + // Copia il contenuto del file originale nel buffer + _, err = io.Copy(&buffer, file) + if err != nil { + return nil, err + } + + // Riapri il file in scrittura + file, err = os.Create(filename) + if err != nil { + return nil, err + } + defer file.Close() + + // Scrivi il contenuto del buffer nel file + _, err = io.Copy(file, &buffer) + if err != nil { + return nil, err + } + + return meta, nil +} diff --git a/store/file/file_test.go b/store/file/file_test.go index e68ba7a..1728ebb 100644 --- a/store/file/file_test.go +++ b/store/file/file_test.go @@ -46,7 +46,7 @@ Question text (3). }, } - quiz, err := QuizFromMarkdown(markdown) + quiz, _, err := QuizFromMarkdown(markdown) t.Nil(err, fmt.Sprintf("Quiz should be parsed without errors: %v", err)) if !t.Failed() { @@ -65,9 +65,9 @@ func (t *testSuite) TestReadAllQuizzes() { if !t.Failed() { t.Equal( - 2, + 4, len(result), - fmt.Sprintf("The store contains 3 files but only 2 should be parsed (duplicated quiz). Total of parsed quizzes are instead %v", len(result)), + fmt.Sprintf("The store contains 5 files but only 4 should be parsed (duplicated quiz). Total of parsed quizzes are instead %v", len(result)), ) } } @@ -136,7 +136,7 @@ func (t *testSuite) TestCreateQuiz() { if !t.Failed() { t.True(exists != nil, "The new quiz file was not created.") if !t.Failed() { - quizFromDisk, err := readQuizFromDisk(path) + quizFromDisk, _, err := readQuizFromDisk(path) t.Nil(err, "Quiz should be read from disk without errors.") if !t.Failed() { t.True(reflect.DeepEqual(quizFromDisk, clientQuiz), "Quiz read from disk and stored in memory should be equal.") @@ -205,7 +205,7 @@ func (t *testSuite) TestUpdateQuiz() { if !t.Failed() { clientQuiz := &client.Quiz{ - Question: &client.Question{Text: "Newly created question text."}, + Question: &client.Question{Text: "Updated question text."}, Answers: []*client.Answer{ {Text: "Answer 1", Correct: true}, {Text: "Answer 2", Correct: false}, @@ -220,7 +220,7 @@ func (t *testSuite) TestUpdateQuiz() { }, quiz.ID) t.Nil(err, fmt.Sprintf("Quiz should be updated without errors: %v", err)) - // t.Equal(updatedQuiz.ID, quiz.ID, "Quiz ID should remain the same") + t.Equal(updatedQuiz.ID, quiz.ID, fmt.Sprintf("IDs should remain the same after an update: updated ID %v original ID %v", updatedQuiz.ID, quiz.ID)) if !t.Failed() { path, err := store.GetPath(updatedQuiz) @@ -229,7 +229,7 @@ func (t *testSuite) TestUpdateQuiz() { t.Nil(err, "GetPath should not raise an error.") if !t.Failed() { - quizFromDisk, err := readQuizFromDisk(path) + quizFromDisk, _, err := readQuizFromDisk(path) t.Nil(err, "Quiz should be read from disk without errors.") if !t.Failed() { t.True(reflect.DeepEqual(clientQuiz, quizFromDisk), "Quiz should be updated.") @@ -244,15 +244,52 @@ func (t *testSuite) TestUpdateQuiz() { } +func (t *testSuite) TestReadMetaHeaderFromFile() { + dirname := "./testdata/quizzes" + store, err := NewFileProboCollectorStore(dirname) + t.True(err == nil, fmt.Sprintf("A file store should be initialized without problems but an error occurred: %v", err)) + + meta, err := store.ReadMetaHeaderFromFile("quiz_4.md") + t.True(err == nil, fmt.Sprintf("An error occurred: %v", err)) + if !t.Failed() { + t.True(meta.ID != "") + t.True(meta.CreatedAt.String() != "") + } +} + +func (t *testSuite) TestWriteMetaHeaderToFile() { + dirname := "./testdata/quizzes" + store, err := NewFileProboCollectorStore(dirname) + + t.True(err == nil, fmt.Sprintf("A file store should be initialized without problems but an error occurred: %v", err)) + + if !t.Failed() { + meta, err := store.ReadMetaHeaderFromFile("quiz_5.md") + t.True(err == nil) + if !t.Failed() { + t.True(meta != nil, "Meta header should not be nil") + + if !t.Failed() { + t.True(meta.ID != "", "ID should not be empty") + + if !t.Failed() { + _, err = store.removeMetaFromFile("quiz_5.md") + t.True(err == nil) + } + } + } + } +} + func createQuizOnDisk(store *FileProboCollectorStore, req *client.CreateUpdateQuizRequest) (*models.Quiz, error) { return store.CreateQuiz(req) } -func readQuizFromDisk(path string) (*client.Quiz, error) { +func readQuizFromDisk(path string) (*client.Quiz, *models.Meta, error) { content, err := os.ReadFile(path) if err != nil { - return nil, err + return nil, nil, err } return QuizFromMarkdown(string(content)) } diff --git a/store/file/testdata/quizzes/quiz_1.md b/store/file/testdata/quizzes/quiz_1.md index e64f194..9c9293b 100644 --- a/store/file/testdata/quizzes/quiz_1.md +++ b/store/file/testdata/quizzes/quiz_1.md @@ -1,3 +1,7 @@ +--- +id: 42982769-2168-45e8-b307-8764c18bec55 +created_at: !!timestamp 2023-09-22T09:07:47.744571823+02:00 +--- Question text 1. * Answer 1 diff --git a/store/file/testdata/quizzes/quiz_2.md b/store/file/testdata/quizzes/quiz_2.md index 896f6b7..4677398 100644 --- a/store/file/testdata/quizzes/quiz_2.md +++ b/store/file/testdata/quizzes/quiz_2.md @@ -1,3 +1,7 @@ +--- +id: a09045c3-af87-4a83-a2bb-7283a2ac67d6 +created_at: !!timestamp 2023-09-22T09:08:50.366639817+02:00 +--- Question text 2. * Answer 1 diff --git a/store/file/testdata/quizzes/quiz_3.md b/store/file/testdata/quizzes/quiz_3.md index 896f6b7..a6efa33 100644 --- a/store/file/testdata/quizzes/quiz_3.md +++ b/store/file/testdata/quizzes/quiz_3.md @@ -1,3 +1,7 @@ +--- +id: a09045c3-af87-4a83-a2bb-7283a2ac67d6 +created_at: !!timestamp 2023-09-22T09:08:50.366772426+02:00 +--- Question text 2. * Answer 1 diff --git a/store/file/testdata/quizzes/quiz_4.md b/store/file/testdata/quizzes/quiz_4.md new file mode 100644 index 0000000..e8a8415 --- /dev/null +++ b/store/file/testdata/quizzes/quiz_4.md @@ -0,0 +1,10 @@ +--- +id: 0000-1234-5678-9101 +created_at: 2006-01-02T15:04:05Z +--- +This quiz file contains a meta header. + +* Answer 1 +* Answer 2 +* Answer 3 +* Answer 4 diff --git a/store/file/testdata/quizzes/quiz_5.md b/store/file/testdata/quizzes/quiz_5.md new file mode 100644 index 0000000..6331e1a --- /dev/null +++ b/store/file/testdata/quizzes/quiz_5.md @@ -0,0 +1,7 @@ +This quiz is initially without metadata. + +* Answer 1 +* Answer 2 +* Answer 3 +* Answer 4 + diff --git a/store/memory/memory.go b/store/memory/memory.go index 81f79dc..c8a0c18 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -154,7 +154,6 @@ func (s *MemoryProboCollectorStore) CalculateQuizHash(quiz *client.Quiz) string } func (s *MemoryProboCollectorStore) createOrUpdateQuiz(r *client.CreateUpdateQuizRequest, id string) (*models.Quiz, bool, error) { - hashes := s.hasher.QuizHashes(r.Quiz) quizHash := hashes[len(hashes)-1] @@ -163,13 +162,19 @@ func (s *MemoryProboCollectorStore) createOrUpdateQuiz(r *client.CreateUpdateQui return quiz, false, nil } - if id != "" { + if id != "" { // we're updating a quiz quiz = s.getQuizFromID(id) if quiz == nil { // Quiz is not present in the store return nil, false, fmt.Errorf("Quiz ID %v doesn't exist in the store!", id) } } else { - id = uuid.New().String() + if r.Meta != nil { + if r.Meta.ID != "" { + id = r.Meta.ID + } + } else { + id = uuid.New().String() + } quiz = new(models.Quiz) }