1
Fork 0

Add initial files

This commit is contained in:
Joshua Goins 2022-06-23 14:51:29 -04:00
commit fdcf443d9d
5 changed files with 276 additions and 0 deletions

10
LICENSE Normal file
View file

@ -0,0 +1,10 @@
The MIT License
Copyright 2022 Joshua Goins
Copyright 2019 awused (based off of their rss-scrapers project)
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

7
README.md Normal file
View file

@ -0,0 +1,7 @@
# amiamirss
This is a simple AmiAmi scraper used to generate a RSS feed. It's too basic at the moment and doesn't keep track of any state.
## Licensing
This is originally based off of https://github.com/awused/rss-scrapers but modified to instead read the "New Items" page instead of a specific search page.

25
src/go.mod Normal file
View file

@ -0,0 +1,25 @@
module git.sr.ht/~redstrate/amiamirss
go 1.12
require (
github.com/PuerkitoBio/goquery v1.7.0 // indirect
github.com/andybalholm/cascadia v1.2.0 // indirect
github.com/antchfx/htmlquery v1.2.3
github.com/antchfx/xpath v1.2.0 // indirect
github.com/awused/awconf v0.0.0-20210616091321-7189f311cc83
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gorilla/feeds v1.1.1
github.com/json-iterator/go v1.1.11 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/mmcdole/gofeed v1.1.3
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.5.1 // indirect
github.com/syndtr/goleveldb v1.0.0
golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
)

73
src/go.sum Normal file
View file

@ -0,0 +1,73 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/PuerkitoBio/goquery v1.7.0 h1:O5SP3b9JWqMSVMG69zMfj577zwkSNpxrFf7ybS74eiw=
github.com/PuerkitoBio/goquery v1.7.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/awused/awconf v0.0.0-20210616091321-7189f311cc83/go.mod h1:UwjeQegPaE0Qr4GaOWPba94Fq+diIcArNMw5eMmZi8M=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

161
src/main.go Normal file
View file

@ -0,0 +1,161 @@
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
"github.com/gorilla/feeds"
)
const searchURL = "https://api.amiami.com/api/v1.0/items?pagemax=50&lang=eng&s_st_list_preorder_available=1&s_st_list_newitem_available=1&s_cate_tag=1"
const itemURLPrefix = "https://www.amiami.com/eng/detail/?gcode="
func main() {
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
log.Panic(err)
}
req.Header.Add("X-User-Key", "amiami_dev")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Panic(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Panic(err)
}
var b response
err = json.Unmarshal(body, &b)
if err != nil {
log.Panic(err)
}
if !b.RSuccess {
log.Panic("AmiAmi search failure " + b.RMessage)
}
feed := &feeds.Rss{&feeds.Feed{
Title: "AmiAmi - New items",
Link: &feeds.Link{Href: searchURL},
Description: "AmiAmi - New items",
}}
now := time.Now()
for _, p := range b.Items {
if p.OrderClosedFlg == 1 {
continue
}
feed.Items = append(feed.Items, &feeds.Item{
Title: p.Gname,
Id: p.Gcode,
Link: &feeds.Link{Href: itemURLPrefix + p.Gcode},
Created: now,
})
}
rssFeed := feed.RssFeed()
// 1 hour TTL
rssFeed.Ttl = 60
feedXML, err := xml.Marshal(rssFeed.FeedXml())
if err != nil {
log.Panic(err)
}
fmt.Print(string(feedXML))
}
type response struct {
RSuccess bool `json:"RSuccess"`
RValue interface{} `json:"RValue"`
RMessage string `json:"RMessage"`
SearchResult struct {
TotalResults int `json:"total_results"`
} `json:"search_result"`
Items []struct {
Gcode string `json:"gcode"`
Gname string `json:"gname"`
ThumbURL string `json:"thumb_url"`
ThumbAlt string `json:"thumb_alt"`
ThumbTitle string `json:"thumb_title"`
MinPrice int `json:"min_price"`
MaxPrice int `json:"max_price"`
CPriceTaxed int `json:"c_price_taxed"`
MakerName string `json:"maker_name"`
Saleitem int `json:"saleitem"`
ConditionFlg int `json:"condition_flg"`
ListPreorderAvailable int `json:"list_preorder_available"`
ListBackorderAvailable int `json:"list_backorder_available"`
ListStoreBonus int `json:"list_store_bonus"`
ListAmiamiLimited int `json:"list_amiami_limited"`
InstockFlg int `json:"instock_flg"`
OrderClosedFlg int `json:"order_closed_flg"`
ElementID interface{} `json:"element_id"`
Salestatus string `json:"salestatus"`
SalestatusDetail string `json:"salestatus_detail"`
Releasedate string `json:"releasedate"`
Jancode string `json:"jancode"`
Preorderitem int `json:"preorderitem"`
Saletopitem int `json:"saletopitem"`
ResaleFlg int `json:"resale_flg"`
PreownedSaleFlg interface{} `json:"preowned_sale_flg"`
ForWomenFlg int `json:"for_women_flg"`
GenreMoe int `json:"genre_moe"`
Cate6 interface{} `json:"cate6"`
Cate7 interface{} `json:"cate7"`
BuyFlg int `json:"buy_flg"`
BuyPrice int `json:"buy_price"`
BuyRemarks interface{} `json:"buy_remarks"`
StockFlg int `json:"stock_flg"`
ImageOn int `json:"image_on"`
ImageCategory string `json:"image_category"`
ImageName string `json:"image_name"`
Metaalt interface{} `json:"metaalt"`
} `json:"items"`
Embedded struct {
CategoryTags []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"category_tags"`
Makers []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"makers"`
SeriesTitles []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"series_titles"`
OriginalTitles []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"original_titles"`
CharacterNames []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"character_names"`
Elements []struct {
ID int `json:"id"`
Name string `json:"name"`
Count int `json:"count"`
} `json:"elements"`
} `json:"_embedded"`
}