This is an automated email from the git hooks/post-receive script. New commit to branch bow-v2-go in repository bow. See https://gitlab.nuiton.org/chorem/bow.git commit 636fef8a9c3adb450b50e929156c5a7a00996cdc Author: Benjamin <poussin@codelutin.com> Date: Wed Jan 20 01:34:57 2021 +0100 remise en place de la recuperation du favicon et d'un screenshot --- README.md | 3 + docker/docker-compose.yml | 44 ++++++ go.mod | 1 + go.sum | 2 + migrate/005_create_pageInfo_table.sql | 46 ++++++ pkg/constant/const.go | 9 +- pkg/http/bookmarkResource.go | 9 +- pkg/http/router.go | 8 +- pkg/repository/bookmarkRepository.go | 39 +++-- pkg/repository/pageInfoRepository.go | 34 +++++ pkg/utils/ico.go | 265 ++++++++++++++++++++++++++++++++++ pkg/utils/image.go | 102 +++++++++++++ pkg/utils/log.go | 11 +- run-dev | 5 + web/src/components/Bookmark.vue | 10 +- 15 files changed, 560 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 8294168..e8cac72 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,6 @@ yarn serve - utiliser https://github.com/mozilla/readability et https://github.com/cure53/DOMPurify pour récupérer l'artible de la page et non pas toute la page pour la stocker lorsqu'on la met en bookmark (voir si c'est un service externe ou directement dans bow (plutot solution externe) - utiliser https://github.com/sensepost/gowitness pour generer les screenshots (en faire un service externe dockeriser appelé par bow) + gowitness --debug --disable-db --resolution-x 596 --resolution-y 842 server + convert /tmp/cl3.png -scale 65x91 /tmp/cl3mini.png + \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ce990dc..23d392f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -27,6 +27,8 @@ services: DATABASE_URL: 'postgres://dbuser:xxxxxxxx@database:5432/bow' SECRET_KEY: $BOW_SECRET_KEY BOW_PUBLIC_URL: https://bookmarks.cl + BOW_FAVICON_URL: http://favicon:8080/icon?size=16..24..32&url={url} + BOW_SCREENSHOT_URL: http://screenshot:7171?url={url} logging: driver: 'json-file' options: @@ -112,6 +114,48 @@ services: - prometheus.type=db - prometheus.port=9187 + favicon: + image: matthiasluedtke/iconserver:latest + environment: + SERVER_MODE: download + logging: + driver: 'json-file' + options: + max-file: 5 + max-size: 10m + deploy: + placement: + constraints: + - node.labels.function == saas + resources: + # limits: + # cpus: '0.60' + # memory: 200M + reservations: + cpus: '0.02' + memory: 13M + + screenshoot: + image: leonjza/gowitness:latest + command: ["gowitness", "--disable-db", "--resolution-x", "800", "--resolution-y", "1130", "server", "--address", ":7171"] + logging: + driver: 'json-file' + options: + max-file: 5 + max-size: 10m + deploy: + placement: + constraints: + - node.labels.function == saas + resources: + # limits: + # cpus: '0.60' + # memory: 200M + reservations: + cpus: '0.02' + memory: 13M + + # pgadmin: # image: registry.nuiton.org/codelutin/admsys/swarm-stack/pgadmin:latest # networks: diff --git a/go.mod b/go.mod index 8478305..f45f5a0 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/stretchr/testify v1.7.0 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad + golang.org/x/image v0.0.0-20201208152932-35266b937fa6 golang.org/x/text v0.3.5 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 8ecd903..acf95d7 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6 h1:nfeHNc1nAqecKCy2FCy4HY+soOOe5sDLJ/gZLbx6GYI= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= diff --git a/migrate/005_create_pageInfo_table.sql b/migrate/005_create_pageInfo_table.sql new file mode 100644 index 0000000..4927bfc --- /dev/null +++ b/migrate/005_create_pageInfo_table.sql @@ -0,0 +1,46 @@ +-- on met les images dans une table a part pour pouvoir mutualise la place qu'elle prenne + +-- penser au nettoyage de cette table, si plus aucun bookmark ne reference cette uri +CREATE TABLE pageInfo ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + uri TEXT UNIQUE, + creationDate TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp, + favicon TEXT, -- base64 image + miniscreenshot TEXT, -- base64 image 65x91 + screenshot TEXT -- base64 image 596x842 +); + +CREATE INDEX pageInfo_uri_idx ON pageInfo(uri); + +-- securite +GRANT SELECT, INSERT, UPDATE, DELETE ON pageInfo TO nobody; +GRANT SELECT ON pageInfo TO person; + +-- migration des données +INSERT INTO pageInfo (uri, favicon, screenshot) + SELECT uri, favicon, screenshot FROM bookmark + WHERE favicon IS NOT NULL AND screenshot IS NOT NULL + ON CONFLICT (uri) DO NOTHING; + +INSERT INTO pageInfo (uri, favicon) + SELECT uri, favicon FROM bookmark + WHERE favicon IS NOT NULL + ON CONFLICT (uri) DO NOTHING; + +INSERT INTO pageInfo (uri, screenshot) + SELECT uri, screenshot FROM bookmark + WHERE screenshot IS NOT NULL + ON CONFLICT (uri) DO NOTHING; + +ALTER TABLE bookmark DROP screenshot; +ALTER TABLE bookmark DROP favicon; + +---- create above / drop below ---- + +ALTER TABLE bookmark ADD screenshot TEXT; +ALTER TABLE bookmark ADD favicon TEXT; + +UPDATE bookmark b SET favicon=i.favicon, screenshot=i.screenshot + FROM pageInfo i WHERE b.uri = i.uri; + +DROP TABLE pageInfo; \ No newline at end of file diff --git a/pkg/constant/const.go b/pkg/constant/const.go index 2ea47d2..38f1e69 100644 --- a/pkg/constant/const.go +++ b/pkg/constant/const.go @@ -6,8 +6,15 @@ import ( "gitlab.chorem.org/chorem/bow/pkg/model" ) +// Debug true si on est en debug var Debug = os.Getenv("BOW_DEBUG") == "true" +// ScreenShotURL l'url du service de screenshot +var ScreenShotURL = os.Getenv("BOW_SCREENSHOT_URL") + +// FaviconURL l'url du service de favicon +var FaviconURL = os.Getenv("BOW_FAVICON_URL") + type httpRequestContexttKey string /* @@ -51,7 +58,7 @@ Action query parameter name for tags search const Action = "action" /* -Action query parameter name for tags search +Suggestion query parameter name for tags search */ const Suggestion = "suggestion" diff --git a/pkg/http/bookmarkResource.go b/pkg/http/bookmarkResource.go index d0f07ce..5ccc0b2 100644 --- a/pkg/http/bookmarkResource.go +++ b/pkg/http/bookmarkResource.go @@ -141,14 +141,15 @@ func addBookmark(w http.ResponseWriter, r *http.Request) { } func deleteBookmark(w http.ResponseWriter, r *http.Request) { + currentUser := r.Context().Value(constant.User).(model.BowUser) + id := mux.Vars(r)["id"] log.Println("Delete bookmark", id) - err := repository.DeleteBookmark(id) + err := repository.DeleteBookmark(currentUser, id) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 400) return } - } /* @@ -174,6 +175,8 @@ func updateBookmark(w http.ResponseWriter, r *http.Request) { } func updateBookmarkAuthenticationInfo(w http.ResponseWriter, r *http.Request) { + currentUser := r.Context().Value(constant.User).(model.BowUser) + id := mux.Vars(r)["id"] var auth model.AuthenticationInfo err := json.NewDecoder(r.Body).Decode(&auth) @@ -182,7 +185,7 @@ func updateBookmarkAuthenticationInfo(w http.ResponseWriter, r *http.Request) { return } - err = repository.UpdateBookmarkAuthenticationInfo(id, auth) + err = repository.UpdateBookmarkAuthenticationInfo(currentUser, id, auth) if err != nil { http.Error(w, fmt.Sprintf("%s", err), 400) return diff --git a/pkg/http/router.go b/pkg/http/router.go index daeb414..730f5e9 100644 --- a/pkg/http/router.go +++ b/pkg/http/router.go @@ -23,6 +23,7 @@ import ( "github.com/gorilla/mux" ) +// BowPublicURL url public var BowPublicURL string var stats = utils.NewStats() @@ -51,7 +52,7 @@ func Start(bowPublicURL string, addr string) { s.HandleFunc("/users/auth", createAuth).Methods(http.MethodPost, http.MethodOptions) u := s.PathPrefix("/users/{id}").Subrouter() - u.Use(convertCurrentToId) + u.Use(convertCurrentToID) u.HandleFunc("", getUser).Methods(http.MethodGet, http.MethodOptions) u.HandleFunc("", deleteUser).Methods(http.MethodDelete, http.MethodOptions) u.HandleFunc("/auth", createAuth).Methods(http.MethodPost, http.MethodOptions) @@ -152,7 +153,7 @@ func needAdminEndpoint(r *http.Request) bool { return result } -func convertCurrentToId(next http.Handler) http.Handler { +func convertCurrentToID(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] if id == "current" { @@ -232,9 +233,8 @@ func authentication(next http.Handler) http.Handler { if err != nil { utils.Throw(w, err) return - } else { - user = tmp } + user = tmp } } else { utils.Throw(w, utils.NewHTTPError("Need authentication", user, 401)) diff --git a/pkg/repository/bookmarkRepository.go b/pkg/repository/bookmarkRepository.go index 0e9684e..81cbca0 100644 --- a/pkg/repository/bookmarkRepository.go +++ b/pkg/repository/bookmarkRepository.go @@ -1,7 +1,6 @@ package repository import ( - "context" "fmt" "strings" "time" @@ -12,6 +11,14 @@ import ( "gitlab.chorem.org/chorem/bow/pkg/utils" ) +func collectPageInfo(uri string) { + favicon := utils.GetFavicon(uri) + mini, maxi := utils.GetScreenShot(uri) + + CreateOrUpdatePageInfo(uri, favicon, mini, maxi) +} + + /* BookmarkJSON retourne le bookmark au format json si id est non vide alors fait une recherche exact sur l'id @@ -43,9 +50,9 @@ func BookmarkJSON(currentUser model.BowUser, id string, uri string, tags []strin utils.LogDebug("search bookmark id: %v, uri: %v, tags: '%v', fulltext: '%v', orderBy: %v, orderAsc: %v, first:%v", id, uri, tags, fulltext, orderBy, orderAsc, first) var mainQuery string if id != "" { - mainQuery = `select * from bookmark where id=$4` + mainQuery = `select b.*, i.favicon, i.miniscreenshot, i.screenshot from bookmark b left join pageInfo i on b.uri=i.uri where b.id=$4` } else if uri != "" { - mainQuery = `select * from bookmark where uri=$3` + mainQuery = `select b.*, i.favicon, i.miniscreenshot, i.screenshot from bookmark b left join pageInfo i on b.uri=i.uri where b.uri=$3` } else { // si fulltext and vide, aucun resultat n'est retourne, ce n'est pas ce qu'on veut // dans ce cas, on souhaite que la recherche ne porte que sur les tags @@ -74,11 +81,11 @@ func BookmarkJSON(currentUser model.BowUser, id string, uri string, tags []strin // vu qu'on veut utiliser l'index fulltext créer, il faut les meme champs dans la requete (to_tsvector) mainQuery = fmt.Sprintf(`select - *%s - FROM bookmark WHERE + b.*, i.favicon, i.miniscreenshot, i.screenshot%s + FROM bookmark b left join pageInfo i on b.uri=i.uri WHERE tags @> string_to_array($1, ',') AND - (%s to_tsvector(lang, text(coalesce(tags, '{}'::text[])) || ' ' || coalesce(description, '') || ' ' || coalesce(uri, '') || ' ' || text(coalesce(privateAlias, '{}'::text[])) || ' ' || text(coalesce(publicAlias, '{}'::text[]))) @@ websearch_to_tsquery($2)) + (%s to_tsvector(lang, text(coalesce(tags, '{}'::text[])) || ' ' || coalesce(description, '') || ' ' || coalesce(b.uri, '') || ' ' || text(coalesce(privateAlias, '{}'::text[])) || ' ' || text(coalesce(publicAlias, '{}'::text[]))) @@ websearch_to_tsquery($2)) `, hl, queryEmptyFulltext) } @@ -106,7 +113,7 @@ func BookmarkJSON(currentUser model.BowUser, id string, uri string, tags []strin BookmarkByIDJSON retourne le bookmark au format json */ func BookmarkByIDJSON(currentUser model.BowUser, id string) (string, error) { - q := &query{sql: `select json_agg(b.*) as result from bookmark b where id=$1`} + q := &query{sql: `select json_agg(b.*, i.favicon, i.miniscreenshot, i.screenshot) as result from bookmark b left join pageInfo i on b.uri=i.uri where b.id=$1`} result, err := q.QueryString(currentUser, id) if err != nil { @@ -200,6 +207,7 @@ func URIFromAlias(currentUser model.BowUser, alias string) (string, error) { return result, nil } + // AddOneVisit ajout 1 au compteur de visite func AddOneVisit(currentUser model.BowUser, id string) (string, error) { q := &query{sql: `update bookmark SET visit = visit + 1 where id=$1 returning uri;`} @@ -241,6 +249,8 @@ func CreateBookmark(currentUser model.BowUser, bookmark model.Bookmark) (string, return "", utils.NewHTTPError500(err, currentUser) } + go collectPageInfo(bookmark.URI) + return id, err } @@ -265,23 +275,25 @@ func UpdateBookmark(currentUser model.BowUser, bookmark model.Bookmark) error { return utils.NewHTTPError500(err, currentUser) } - return err + go collectPageInfo(bookmark.URI) + return err } /* UpdateBookmarkAuthenticationInfo creation d'un nouveau bookmark */ -func UpdateBookmarkAuthenticationInfo(id string, auth model.AuthenticationInfo) error { +func UpdateBookmarkAuthenticationInfo(currentUser model.BowUser, id string, auth model.AuthenticationInfo) error { authAsJSON, err := json.Marshal(auth) if err != nil { return err } - _, err = db.Exec(context.Background(), ` + q := &query{sql: ` UPDATE bookmark AS t SET (authenticationinfo) = (SELECT * FROM json_populate_record(NULL::authenticationinfo, $2::json)) - WHERE id=$1`, id, string(authAsJSON)) + WHERE id=$1`} + err = q.execOnOneRow(currentUser, id, string(authAsJSON)) return err } @@ -289,8 +301,9 @@ func UpdateBookmarkAuthenticationInfo(id string, auth model.AuthenticationInfo) /* DeleteBookmark suppression d'un nouveau bookmark */ -func DeleteBookmark(id string) error { - _, err := db.Exec(context.Background(), `DELETE FROM bookmark WHERE id=$1`, id) +func DeleteBookmark(currentUser model.BowUser, id string) error { + q := &query{sql: `DELETE FROM bookmark WHERE id=$1 RETURNING uri`} + err := q.execOnOneRow(currentUser, id) return err } diff --git a/pkg/repository/pageInfoRepository.go b/pkg/repository/pageInfoRepository.go new file mode 100644 index 0000000..ffa982b --- /dev/null +++ b/pkg/repository/pageInfoRepository.go @@ -0,0 +1,34 @@ +package repository + +import ( + "gitlab.chorem.org/chorem/bow/pkg/constant" + "gitlab.chorem.org/chorem/bow/pkg/utils" +) + +/* +CreateOrUpdatePageInfo creation ou mise a jour des infos +return: id, err +*/ +func CreateOrUpdatePageInfo(uri string, favicon string, mini string, maxi string) { + utils.LogDebug("Update info for %s\n", uri) + + q := &query{sql: ` + INSERT INTO pageInfo AS i (uri, favicon, miniscreenshot, screenshot) values ($1, $2, $3, $4) + ON CONFLICT(uri) DO UPDATE SET favicon = COALESCE(EXCLUDED.favicon, i.favicon), miniscreenshot = COALESCE(EXCLUDED.miniscreenshot, i.miniscreenshot), screenshot = COALESCE(EXCLUDED.screenshot, i.screenshot);`} + err := q.execOnOneRow(constant.Nobody, uri, emptyToNil(favicon), emptyToNil(mini), emptyToNil(maxi)) + + // _, err := db.Exec(context.Background(), ` + // INSERT INTO pageInfo AS i (uri, favicon, miniscreenshot, screenshot) values ($1, $2, $3, $4) + // ON CONFLICT(uri) DO UPDATE SET favicon = COALESCE(EXCLUDED.favicon, i.favicon), miniscreenshot = COALESCE(EXCLUDED.miniscreenshot, i.miniscreenshot), screenshot = COALESCE(EXCLUDED.screenshot, i.screenshot);`, + // uri, emptyToNil(favicon), emptyToNil(mini), emptyToNil(maxi)) + if err != nil { + utils.LogError("can't insert or update pageInfo %v\n", err) + } +} + +func emptyToNil(s string) interface{} { + if s == "" { + return nil + } + return s +} diff --git a/pkg/utils/ico.go b/pkg/utils/ico.go new file mode 100644 index 0000000..2d5b37e --- /dev/null +++ b/pkg/utils/ico.go @@ -0,0 +1,265 @@ +// Package utils ico registers image.Decode and DecodeConfig support +// for the icon (container) format. + +// from https://github.com/mat/besticon/blob/master/ico/ico.go +// original license: MIT License (MIT) + +package utils + +import ( + "bytes" + "encoding/binary" + "errors" + "image" + "io" + "io/ioutil" + + "image/png" + + "golang.org/x/image/bmp" +) + +type icondir struct { + Reserved uint16 + Type uint16 + Count uint16 + Entries []icondirEntry +} + +type icondirEntry struct { + Width byte + Height byte + PaletteCount byte + Reserved byte + ColorPlanes uint16 + BitsPerPixel uint16 + Size uint32 + Offset uint32 +} + +func (dir *icondir) FindBestIcon() *icondirEntry { + if len(dir.Entries) == 0 { + return nil + } + + best := dir.Entries[0] + for _, e := range dir.Entries { + if (e.width() > best.width()) && (e.height() > best.height()) { + best = e + } + } + return &best +} + +// ParseIco parses the icon and returns meta information for the icons as icondir. +func ParseIco(r io.Reader) (*icondir, error) { + dir := icondir{} + + var err error + err = binary.Read(r, binary.LittleEndian, &dir.Reserved) + if err != nil { + return nil, err + } + + err = binary.Read(r, binary.LittleEndian, &dir.Type) + if err != nil { + return nil, err + } + + err = binary.Read(r, binary.LittleEndian, &dir.Count) + if err != nil { + return nil, err + } + + for i := uint16(0); i < dir.Count; i++ { + entry := icondirEntry{} + e := parseIcondirEntry(r, &entry) + if e != nil { + return nil, e + } + dir.Entries = append(dir.Entries, entry) + } + + return &dir, err +} + +func parseIcondirEntry(r io.Reader, e *icondirEntry) error { + err := binary.Read(r, binary.LittleEndian, e) + if err != nil { + return err + } + + return nil +} + +type dibHeader struct { + dibHeaderSize uint32 + width uint32 + height uint32 +} + +func (e *icondirEntry) ColorCount() int { + if e.PaletteCount == 0 { + return 256 + } + return int(e.PaletteCount) +} + +func (e *icondirEntry) width() int { + if e.Width == 0 { + return 256 + } + return int(e.Width) +} + +func (e *icondirEntry) height() int { + if e.Height == 0 { + return 256 + } + return int(e.Height) +} + +// DecodeConfig returns just the dimensions of the largest image +// contained in the icon withou decoding the entire icon file. +func DecodeConfig(r io.Reader) (image.Config, error) { + dir, err := ParseIco(r) + if err != nil { + return image.Config{}, err + } + + best := dir.FindBestIcon() + if best == nil { + return image.Config{}, errInvalid + } + return image.Config{Width: best.width(), Height: best.height()}, nil +} + +// The bitmap header structure we read from an icondirEntry +type bitmapHeaderRead struct { + Size uint32 + Width uint32 + Height uint32 + Planes uint16 + BitCount uint16 + Compression uint32 + ImageSize uint32 + XPixelsPerMeter uint32 + YPixelsPerMeter uint32 + ColorsUsed uint32 + ColorsImportant uint32 +} + +// The bitmap header structure we need to generate for bmp.Decode() +type bitmapHeaderWrite struct { + sigBM [2]byte + fileSize uint32 + resverved [2]uint16 + pixOffset uint32 + Size uint32 + Width uint32 + Height uint32 + Planes uint16 + BitCount uint16 + Compression uint32 + ImageSize uint32 + XPixelsPerMeter uint32 + YPixelsPerMeter uint32 + ColorsUsed uint32 + ColorsImportant uint32 +} + +var errInvalid = errors.New("ico: invalid ICO image") + +// Decode returns the largest image contained in the icon +// which might be a bmp or png +func Decode(r io.Reader) (image.Image, error) { + icoBytes, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + r = bytes.NewReader(icoBytes) + dir, err := ParseIco(r) + if err != nil { + return nil, errInvalid + } + + best := dir.FindBestIcon() + if best == nil { + return nil, errInvalid + } + + return parseImage(best, icoBytes) +} + +func parseImage(entry *icondirEntry, icoBytes []byte) (image.Image, error) { + r := bytes.NewReader(icoBytes) + r.Seek(int64(entry.Offset), 0) + + // Try PNG first then BMP + img, err := png.Decode(r) + if err != nil { + return parseBMP(entry, icoBytes) + } + return img, nil +} + +func parseBMP(entry *icondirEntry, icoBytes []byte) (image.Image, error) { + bmpBytes, err := makeFullBMPBytes(entry, icoBytes) + if err != nil { + return nil, err + } + return bmp.Decode(bmpBytes) +} + +func makeFullBMPBytes(entry *icondirEntry, icoBytes []byte) (*bytes.Buffer, error) { + r := bytes.NewReader(icoBytes) + r.Seek(int64(entry.Offset), 0) + + var err error + h := bitmapHeaderRead{} + + err = binary.Read(r, binary.LittleEndian, &h) + if err != nil { + return nil, err + } + + if h.Size != 40 || h.Planes != 1 { + return nil, errInvalid + } + + var pixOffset uint32 + if h.ColorsUsed == 0 && h.BitCount <= 8 { + pixOffset = 14 + 40 + 4*(1<<h.BitCount) + } else { + pixOffset = 14 + 40 + 4*h.ColorsUsed + } + + writeHeader := &bitmapHeaderWrite{ + sigBM: [2]byte{'B', 'M'}, + fileSize: 14 + 40 + uint32(len(icoBytes)), // correct? important? + pixOffset: pixOffset, + Size: 40, + Width: uint32(h.Width), + Height: uint32(h.Height / 2), + Planes: h.Planes, + BitCount: h.BitCount, + Compression: h.Compression, + ColorsUsed: h.ColorsUsed, + ColorsImportant: h.ColorsImportant, + } + + buf := new(bytes.Buffer) + if err = binary.Write(buf, binary.LittleEndian, writeHeader); err != nil { + return nil, err + } + io.CopyN(buf, r, int64(entry.Size)) + + return buf, nil +} + +const icoHeader = "\x00\x00\x01\x00" + +func init() { + image.RegisterFormat("ico", icoHeader, Decode, DecodeConfig) +} diff --git a/pkg/utils/image.go b/pkg/utils/image.go new file mode 100644 index 0000000..1096056 --- /dev/null +++ b/pkg/utils/image.go @@ -0,0 +1,102 @@ +package utils + +import ( + "bytes" + "encoding/base64" + "image" + // on veut le support des gif + _ "image/gif" + // on veut le support des jpeg + _ "image/jpeg" + "image/png" + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" + + "gitlab.chorem.org/chorem/bow/pkg/constant" + "golang.org/x/image/draw" +) + +// GetScreenShot recupere le screenshot de la page +func GetScreenShot(uri string) (string, string) { + if constant.ScreenShotURL == "" { + return "", "" + } + + ssu := strings.ReplaceAll(constant.ScreenShotURL, "{url}", url.QueryEscape(uri)) + log.Println("get screenshot from ", ssu) + resp, err := http.Get(ssu) + if err != nil { + log.Printf("[error] %v\n", err) + return "", "" + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("[error] %v\n", err) + return "", "" + } + + buffer := bytes.NewBuffer(body) + loadedImage, _, err := image.Decode(buffer) + + mini := resize(loadedImage, 65, 91) + maxi := base64.StdEncoding.EncodeToString(body) + + return mini, maxi +} + +func resize(src image.Image, x int, y int) string { + rect := image.Rect(0, 0, x, y) + // resize using given scaler + dst := image.NewRGBA(rect) + draw.NearestNeighbor.Scale(dst, rect, src, src.Bounds(), draw.Over, nil) + + var buff bytes.Buffer + err := png.Encode(&buff, dst) + + if err != nil { + return "" + } + + encodedString := base64.StdEncoding.EncodeToString(buff.Bytes()) + return encodedString + +} + +// GetFavicon recupere le favicon de la page +func GetFavicon(uri string) string { + if constant.FaviconURL == "" { + return "" + } + + ssu := strings.ReplaceAll(constant.FaviconURL, "{url}", url.QueryEscape(uri)) + log.Println("get favicon from ", ssu) + resp, err := http.Get(ssu) + if err != nil { + log.Printf("[error] %v\n", err) + return "" + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "" + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("[error] %v\n", err) + return "" + } + + buffer := bytes.NewBuffer(body) + loadedImage, _, err := image.Decode(buffer) + + fav := resize(loadedImage, 24, 24) + + return fav +} diff --git a/pkg/utils/log.go b/pkg/utils/log.go index 42fe58e..3a9823d 100644 --- a/pkg/utils/log.go +++ b/pkg/utils/log.go @@ -11,7 +11,7 @@ LogDebug log if debug level */ func LogDebug(msg string, a ...interface{}) { if constant.Debug { - log.Printf(msg, a...) + log.Printf("[DBG] " + msg, a...) } } @@ -19,6 +19,13 @@ func LogDebug(msg string, a ...interface{}) { Log log all time */ func Log(msg string, a ...interface{}) { - log.Printf(msg, a...) + log.Printf("[LOG] " + msg, a...) +} + +/* +LogError log all time +*/ +func LogError(msg string, a ...interface{}) { + log.Printf("[ERR] " + msg, a...) } diff --git a/run-dev b/run-dev index 2adc709..d4c59a6 100755 --- a/run-dev +++ b/run-dev @@ -1,4 +1,9 @@ +# docker run -ti --rm -e SERVER_MODE=download -p 8080:8080 matthiasluedtke/iconserver:latest +# docker run -ti --rm -p 7171:7171 leonjza/gowitness:latest gowitness --disable-db --resolution-x 596 --resolution-y 842 server + export DATABASE_URL=postgres://dbuser:xxxxxxxx@localhost:5432/bow export SECRET_KEY="AZERTYUIOPQSDFGHJKLMWXCVBN" export BOW_PUBLIC_URL="http://localhost:8000" +export BOW_FAVICON_URL="http://localhost:8080/icon?size=16..24..32&url={url}" +export BOW_SCREENSHOT_URL="http://localhost:7171?url={url}" go run cmd/bow/main.go diff --git a/web/src/components/Bookmark.vue b/web/src/components/Bookmark.vue index 07fad81..65348f8 100644 --- a/web/src/components/Bookmark.vue +++ b/web/src/components/Bookmark.vue @@ -8,11 +8,15 @@ class="bookmark-screenshot" :bookmarkId="bookmark.id" :link="bookmark.uri"> - <img v-if="bookmark.screenshot" :src="hexaToImg(bookmark.screenshot)" /> + <img v-if="bookmark.miniscreenshot" :src="hexaToImg(bookmark.miniscreenshot)" /> </LinkCount> <div class="bookmark-info"> <div class="bookmark-title"> + <LinkCount :bookmarkId="bookmark.id" :link="bookmark.uri"> + <img v-if="bookmark.favicon" :src="hexaToImg(bookmark.favicon)" /> + </LinkCount> + <LinkCount class="bookmark-uri" :bookmarkId="bookmark.id" @@ -20,10 +24,6 @@ {{ bookmark.uri }} </LinkCount> - <LinkCount :bookmarkId="bookmark.id" :link="bookmark.uri"> - <img v-if="bookmark.favicon" :src="hexaToImg(bookmark.favicon)" /> - </LinkCount> - <Visit :visit="bookmark.visit"></Visit> </div> -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.