andrea/probo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add metadata headers

Browse source
This commit is contained in:
andrea 2023-09-22 10:29:10 +02:00
parent 9780956432
commit 4f3ecda14c
13 changed files with 358 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
client/client.go
View file

@ -46,6 +46,7 @@ type CreateAnswerRequest struct {
type CreateUpdateQuizRequest struct {
*Quiz
*models.Meta
}
type DeleteQuizRequest struct {

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.17
require github.com/sirupsen/logrus v1.8.1
require (
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/kr/pretty v0.2.1 // indirect

2
go.sum
View file

@ -1,4 +1,6 @@
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=

8
models/meta.go Normal file
View file

@ -0,0 +1,8 @@
package models
import "time"
type Meta struct {
ID string `json:"id" yaml:"id"`
CreatedAt time.Time `json:"created_at" yaml:"created_at"`
}

3
models/quiz.go
View file

@ -1,7 +1,8 @@
package models
type Quiz struct {
ID string `json:"id"`
Meta
// ID string `json:"id"`
Hash string `json:"hash"`
Question *Question `json:"question"`
Answers []*Answer `json:"answers"`

269
store/file/file.go
View file

@ -1,11 +1,15 @@
package file
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"sync"
@ -15,8 +19,11 @@ import (
"git.andreafazzi.eu/andrea/probo/hasher/sha256"
"git.andreafazzi.eu/andrea/probo/models"
"git.andreafazzi.eu/andrea/probo/store/memory"
"github.com/go-yaml/yaml"
)
var ErrorMetaHeaderIsNotPresent = errors.New("Meta header was not found in file.")
type FileProboCollectorStore struct {
Dir string
@ -73,17 +80,25 @@ func (s *FileProboCollectorStore) Reindex() error {
if err != nil {
return err
}
quiz, err := QuizFromMarkdown(string(content))
quiz, meta, err := QuizFromMarkdown(string(content))
if err != nil {
return err
}
q, err := s.memoryStore.CreateQuiz(&client.CreateUpdateQuizRequest{
Quiz: quiz,
Meta: meta,
})
if err != nil {
return err
}
if meta == nil {
s.WriteMetaHeaderToFile(filename, &models.Meta{
ID: q.ID,
CreatedAt: time.Now(),
})
}
s.SetPath(q, fullPath)
}
@ -202,8 +217,13 @@ func MarkdownFromQuiz(quiz *models.Quiz) (string, error) {
return markdown, nil
}
func QuizFromMarkdown(markdown string) (*client.Quiz, error) {
lines := strings.Split(markdown, "\n")
func QuizFromMarkdown(markdown string) (*client.Quiz, *models.Meta, error) {
meta, remainingMarkdown, err := parseMetaHeaderFromMarkdown(markdown)
if err != nil {
return nil, nil, err
}
lines := strings.Split(remainingMarkdown, "\n")
questionText := ""
answers := []*client.Answer{}
@ -225,17 +245,113 @@ func QuizFromMarkdown(markdown string) (*client.Quiz, error) {
questionText = strings.TrimRight(questionText, "\n")
if questionText == "" {
return nil, fmt.Errorf("Question text should not be empty.")
return nil, nil, fmt.Errorf("Question text should not be empty.")
}
if len(answers) < 2 {
return nil, fmt.Errorf("Number of answers should be at least 2 but parsed answers are %d.", len(answers))
return nil, nil, fmt.Errorf("Number of answers should be at least 2 but parsed answers are %d.", len(answers))
}
question := &client.Question{Text: questionText}
quiz := &client.Quiz{Question: question, Answers: answers}
return quiz, nil
return quiz, meta, nil
}
func (s *FileProboCollectorStore) ReadMetaHeaderFromFile(filename string) (*models.Meta, error) {
data, err := ioutil.ReadFile(path.Join(s.Dir, filename))
if err != nil {
return nil, err
}
meta, _, err := parseMetaHeaderFromMarkdown(string(data))
if err != nil {
return nil, err
}
return meta, nil
}
func (s *FileProboCollectorStore) WriteMetaHeaderToFile(filename string, meta *models.Meta) (*models.Meta, error) {
readMeta, err := s.ReadMetaHeaderFromFile(filename)
if err != nil {
return nil, err
}
if readMeta == nil {
_, err := writeMetaHeader(path.Join(s.Dir, filename), meta)
if err != nil {
return nil, err
}
}
return meta, nil
}
func (s *FileProboCollectorStore) removeMetaFromFile(filename string) (*models.Meta, error) {
file, err := os.Open(path.Join(s.Dir, filename))
if err != nil {
return nil, err
}
defer file.Close()
var buffer bytes.Buffer
reader := bufio.NewReader(file)
var meta models.Meta
var line string
var sb strings.Builder
for {
line, err = reader.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
if strings.TrimSpace(line) == "---" {
break
}
}
for {
line, err = reader.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
if strings.TrimSpace(line) == "---" {
break
}
sb.WriteString(line)
}
err = yaml.Unmarshal([]byte(sb.String()), &meta)
if err != nil {
return nil, err
}
_, err = io.Copy(&buffer, reader)
if err != nil {
return nil, err
}
file, err = os.Create(path.Join(s.Dir, filename))
if err != nil {
return nil, err
}
defer file.Close()
_, err = io.Copy(file, &buffer)
if err != nil {
return nil, err
}
return &meta, nil
}
func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz) error {
@ -253,10 +369,14 @@ func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write([]byte(markdown))
markdownWithMetaHeader, err := addMetaHeaderToMarkdown(markdown, &quiz.Meta)
if err != nil {
return err
}
_, err = file.Write([]byte(markdownWithMetaHeader))
if err != nil {
return err
}
@ -265,3 +385,136 @@ func (s *FileProboCollectorStore) createOrUpdateMarkdownFile(quiz *models.Quiz)
return nil
}
func addMetaHeaderToMarkdown(content string, meta *models.Meta) (string, error) {
var buffer bytes.Buffer
header, err := yaml.Marshal(meta)
if err != nil {
return "", err
}
_, err = buffer.WriteString("---\n" + string(header) + "---\n")
if err != nil {
return "", err
}
_, err = buffer.WriteString(content)
if err != nil {
return "", err
}
return buffer.String(), nil
}
func parseMetaHeaderFromMarkdown(markdown string) (*models.Meta, string, error) {
reader := strings.NewReader(markdown)
var sb strings.Builder
var line string
var err error
for {
line, err = readLine(reader)
if err != nil {
if err == io.EOF {
break
}
return nil, "", err
}
if strings.TrimSpace(line) == "---" {
break
}
}
for {
line, err = readLine(reader)
if err != nil {
if err == io.EOF {
break
}
return nil, "", err
}
if strings.TrimSpace(line) == "---" {
break
}
sb.WriteString(line)
}
if sb.String() == "" {
return nil, markdown, nil
}
var meta models.Meta
err = yaml.Unmarshal([]byte(sb.String()), &meta)
if err != nil {
return nil, markdown, err
}
remainingMarkdown := markdown[strings.Index(markdown, "---\n"+sb.String()+"---\n")+len("---\n"+sb.String()+"---\n"):]
return &meta, remainingMarkdown, nil
}
func readLine(reader *strings.Reader) (string, error) {
var sb strings.Builder
for {
r, _, err := reader.ReadRune()
if err != nil {
if err == io.EOF {
return sb.String(), io.EOF
}
return "", err
}
sb.WriteRune(r)
if r == '\n' {
break
}
}
return sb.String(), nil
}
func writeMetaHeader(filename string, meta *models.Meta) (*models.Meta, error) {
// Apri il file in lettura
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
// Crea un buffer in memoria
var buffer bytes.Buffer
// Scrivi l'intestazione YAML nel buffer
header, err := yaml.Marshal(meta)
if err != nil {
return nil, err
}
_, err = buffer.WriteString("---\n" + string(header) + "---\n")
if err != nil {
return nil, err
}
// Copia il contenuto del file originale nel buffer
_, err = io.Copy(&buffer, file)
if err != nil {
return nil, err
}
// Riapri il file in scrittura
file, err = os.Create(filename)
if err != nil {
return nil, err
}
defer file.Close()
// Scrivi il contenuto del buffer nel file
_, err = io.Copy(file, &buffer)
if err != nil {
return nil, err
}
return meta, nil
}

55
store/file/file_test.go
View file

@ -46,7 +46,7 @@ Question text (3).
},
}
quiz, err := QuizFromMarkdown(markdown)
quiz, _, err := QuizFromMarkdown(markdown)
t.Nil(err, fmt.Sprintf("Quiz should be parsed without errors: %v", err))
if !t.Failed() {
@ -65,9 +65,9 @@ func (t *testSuite) TestReadAllQuizzes() {
if !t.Failed() {
t.Equal(
2,
4,
len(result),
fmt.Sprintf("The store contains 3 files but only 2 should be parsed (duplicated quiz). Total of parsed quizzes are instead %v", len(result)),
fmt.Sprintf("The store contains 5 files but only 4 should be parsed (duplicated quiz). Total of parsed quizzes are instead %v", len(result)),
)
}
}
@ -136,7 +136,7 @@ func (t *testSuite) TestCreateQuiz() {
if !t.Failed() {
t.True(exists != nil, "The new quiz file was not created.")
if !t.Failed() {
quizFromDisk, err := readQuizFromDisk(path)
quizFromDisk, _, err := readQuizFromDisk(path)
t.Nil(err, "Quiz should be read from disk without errors.")
if !t.Failed() {
t.True(reflect.DeepEqual(quizFromDisk, clientQuiz), "Quiz read from disk and stored in memory should be equal.")
@ -205,7 +205,7 @@ func (t *testSuite) TestUpdateQuiz() {
if !t.Failed() {
clientQuiz := &client.Quiz{
Question: &client.Question{Text: "Newly created question text."},
Question: &client.Question{Text: "Updated question text."},
Answers: []*client.Answer{
{Text: "Answer 1", Correct: true},
{Text: "Answer 2", Correct: false},
@ -220,7 +220,7 @@ func (t *testSuite) TestUpdateQuiz() {
}, quiz.ID)
t.Nil(err, fmt.Sprintf("Quiz should be updated without errors: %v", err))
// t.Equal(updatedQuiz.ID, quiz.ID, "Quiz ID should remain the same")
t.Equal(updatedQuiz.ID, quiz.ID, fmt.Sprintf("IDs should remain the same after an update: updated ID %v original ID %v", updatedQuiz.ID, quiz.ID))
if !t.Failed() {
path, err := store.GetPath(updatedQuiz)
@ -229,7 +229,7 @@ func (t *testSuite) TestUpdateQuiz() {
t.Nil(err, "GetPath should not raise an error.")
if !t.Failed() {
quizFromDisk, err := readQuizFromDisk(path)
quizFromDisk, _, err := readQuizFromDisk(path)
t.Nil(err, "Quiz should be read from disk without errors.")
if !t.Failed() {
t.True(reflect.DeepEqual(clientQuiz, quizFromDisk), "Quiz should be updated.")
@ -244,15 +244,52 @@ func (t *testSuite) TestUpdateQuiz() {
}
func (t *testSuite) TestReadMetaHeaderFromFile() {
dirname := "./testdata/quizzes"
store, err := NewFileProboCollectorStore(dirname)
t.True(err == nil, fmt.Sprintf("A file store should be initialized without problems but an error occurred: %v", err))
meta, err := store.ReadMetaHeaderFromFile("quiz_4.md")
t.True(err == nil, fmt.Sprintf("An error occurred: %v", err))
if !t.Failed() {
t.True(meta.ID != "")
t.True(meta.CreatedAt.String() != "")
}
}
func (t *testSuite) TestWriteMetaHeaderToFile() {
dirname := "./testdata/quizzes"
store, err := NewFileProboCollectorStore(dirname)
t.True(err == nil, fmt.Sprintf("A file store should be initialized without problems but an error occurred: %v", err))
if !t.Failed() {
meta, err := store.ReadMetaHeaderFromFile("quiz_5.md")
t.True(err == nil)
if !t.Failed() {
t.True(meta != nil, "Meta header should not be nil")
if !t.Failed() {
t.True(meta.ID != "", "ID should not be empty")
if !t.Failed() {
_, err = store.removeMetaFromFile("quiz_5.md")
t.True(err == nil)
}
}
}
}
}
func createQuizOnDisk(store *FileProboCollectorStore, req *client.CreateUpdateQuizRequest) (*models.Quiz, error) {
return store.CreateQuiz(req)
}
func readQuizFromDisk(path string) (*client.Quiz, error) {
func readQuizFromDisk(path string) (*client.Quiz, *models.Meta, error) {
content, err := os.ReadFile(path)
if err != nil {
return nil, err
return nil, nil, err
}
return QuizFromMarkdown(string(content))
}

4
store/file/testdata/quizzes/quiz_1.md vendored
View file

@ -1,3 +1,7 @@
---
id: 42982769-2168-45e8-b307-8764c18bec55
created_at: !!timestamp 2023-09-22T09:07:47.744571823+02:00
---
Question text 1.
* Answer 1

4
store/file/testdata/quizzes/quiz_2.md vendored
View file

@ -1,3 +1,7 @@
---
id: a09045c3-af87-4a83-a2bb-7283a2ac67d6
created_at: !!timestamp 2023-09-22T09:08:50.366639817+02:00
---
Question text 2.
* Answer 1

4
store/file/testdata/quizzes/quiz_3.md vendored
View file

@ -1,3 +1,7 @@
---
id: a09045c3-af87-4a83-a2bb-7283a2ac67d6
created_at: !!timestamp 2023-09-22T09:08:50.366772426+02:00
---
Question text 2.
* Answer 1

10
store/file/testdata/quizzes/quiz_4.md vendored Normal file
View file

@ -0,0 +1,10 @@
---
id: 0000-1234-5678-9101
created_at: 2006-01-02T15:04:05Z
---
This quiz file contains a meta header.
* Answer 1
* Answer 2
* Answer 3
* Answer 4

7
store/file/testdata/quizzes/quiz_5.md vendored Normal file
View file

@ -0,0 +1,7 @@
This quiz is initially without metadata.
* Answer 1
* Answer 2
* Answer 3
* Answer 4

9
store/memory/memory.go
View file

@ -154,7 +154,6 @@ func (s *MemoryProboCollectorStore) CalculateQuizHash(quiz *client.Quiz) string
}
func (s *MemoryProboCollectorStore) createOrUpdateQuiz(r *client.CreateUpdateQuizRequest, id string) (*models.Quiz, bool, error) {
hashes := s.hasher.QuizHashes(r.Quiz)
quizHash := hashes[len(hashes)-1]
@ -163,13 +162,19 @@ func (s *MemoryProboCollectorStore) createOrUpdateQuiz(r *client.CreateUpdateQui
return quiz, false, nil
}
if id != "" {
if id != "" { // we're updating a quiz
quiz = s.getQuizFromID(id)
if quiz == nil { // Quiz is not present in the store
return nil, false, fmt.Errorf("Quiz ID %v doesn't exist in the store!", id)
}
} else {
if r.Meta != nil {
if r.Meta.ID != "" {
id = r.Meta.ID
}
} else {
id = uuid.New().String()
}
quiz = new(models.Quiz)
}