Skip to content

Commit

Permalink
Added Support for asset data extraction
Browse files Browse the repository at this point in the history
  • Loading branch information
@almeida-raphael
almeida-raphael committed Jul 4, 2020
1 parent 81e872c commit caa82fd
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 53 deletions.
30 changes: 15 additions & 15 deletions history/get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,46 @@ import (
)

// Get returns a list of infos for assets in B3
func GetByYear(year uint) ([]AssetInfo, error) {
func GetByYear(year uint) (map[string]*Asset, []Price, error) {
responseData, err := download(fmt.Sprintf("http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_A%v.ZIP", year))
if err != nil {
return nil, err
return nil, nil, err
}
encodedContent, err := zip.ExtractFirstFile(responseData)
if err != nil {
return nil, err
return nil, nil, err
}
data, err := parseHistoricDataFromBytes(encodedContent, int(year))
assets, prices, err := parseHistoricDataFromBytes(encodedContent, int(year))
if err != nil {
return nil, err
return nil, nil, err
}

return data, nil
return assets, prices, nil
}

func GetSpecificDay(day, month, year uint) ([]AssetInfo, error) {
func GetSpecificDay(day, month, year uint) (map[string]*Asset, []Price, error) {
if day > 31 || day < 1 {
return nil, &DayOutOfRangeError{Day: day}
return nil, nil, &DayOutOfRangeError{Day: day}
}
if month > 12 || month < 1 {
return nil, &MonthOutOfRangeError{Month: month}
return nil, nil, &MonthOutOfRangeError{Month: month}
}
if year < 2000 {
return nil, &YearOutOfRangeError{Year: year}
return nil, nil, &YearOutOfRangeError{Year: year}
}

responseData, err := download(fmt.Sprintf("http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_D%02d%02d%d.ZIP", day, month, year))
if err != nil {
return nil, err
return nil, nil, err
}
encodedContent, err := zip.ExtractFirstFile(responseData)
if err != nil {
return nil, err
return nil, nil, err
}
data, err := parseHistoricDataFromBytes(encodedContent, int(year))
assets, prices, err := parseHistoricDataFromBytes(encodedContent, int(year))
if err != nil {
return nil, err
return nil, nil, err
}

return data, nil
return assets, prices, nil
}
16 changes: 8 additions & 8 deletions history/history_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,25 @@ package history
import "testing"

func TestHistoryByYear(t *testing.T) {
assets, err := GetByYear(2019)
assets, prices, err := GetByYear(2019)
if err != nil {
t.Fatal(err)
}

for _, asset := range assets {
t.Logf("%+v", asset)
t.Logf("%+v", assets)
for _, price := range prices {
t.Logf("%+v", price)
}

}

func TestHistorySpecificDay(t *testing.T) {
assets, err := GetSpecificDay(30, 9, 2019)
assets, prices, err := GetSpecificDay(30, 9, 2019)
if err != nil {
t.Fatal(err)
}

for _, asset := range assets {
t.Logf("%+v", asset)
t.Logf("%+v", assets)
for _, price := range prices {
t.Logf("%+v", price)
}

}
19 changes: 19 additions & 0 deletions history/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,22 @@ type AssetInfo struct {
ISINCode string `json:"-" bson:"-"`
DistributionNumber int `json:"-" bson:"-"`
}

type Price struct {
Year int `json:"year" bson:"year"`
DataCollectionDate time.Time `json:"data" bson:"date"`
Ticker string `json:"ticker" bson:"ticker"`
FutureMarketExpiration string `json:" future_market_expiration" bson:" future_market_expiration"` // Need to check this against an example that actually has a value
PriceClose FixedPoint `json:"price_close" bson:"price_close"`
TotalVolume FixedPoint `json:"total_volume" bson:"total_volume"`
PreExe FixedPoint `json:"execution_price" bson:"execution_price"` // Needs further investigation
ExpirationDate time.Time `json:"expiration_date" bson:"expiration_date"`
}

type Asset struct {
Ticker string `json:"ticker" bson:"ticker"`
CompanyName string `json:"shot_company_name" bson:"shot_company_name"`
MarketType int `json:"market_type" bson:"market_type"`
SecurityType string `json:"security_type" bson:"security_type"`
Currency string `json:"currency" bson:"currency"`
}
81 changes: 51 additions & 30 deletions history/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,88 +7,89 @@ import (
"time"
)

func parseContentLine(rawLine string, year int) (*AssetInfo, error) {
func parseContentLine(rawLine string, year int) (*Asset, *Price, error) {
tipReg, err := strconv.ParseInt(rawLine[:2], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
date, err := time.Parse("20060102 -0700", fmt.Sprintf("%s -0300", rawLine[2:2+8]))
if err != nil {
return nil, err
return nil, nil, err
}
bdiCode, err := strconv.ParseInt(rawLine[10:10+2], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
marketType, err := strconv.ParseInt(rawLine[24:24+3], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceOpen, err := strconv.ParseInt(rawLine[56:56+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceMax, err := strconv.ParseInt(rawLine[69:69+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceMin, err := strconv.ParseInt(rawLine[82:82+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceMean, err := strconv.ParseInt(rawLine[95:95+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceClose, err := strconv.ParseInt(rawLine[108:108+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceBid, err := strconv.ParseInt(rawLine[121:121+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
priceAsk, err := strconv.ParseInt(rawLine[134:134+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
totalTrades, err := strconv.ParseInt(rawLine[147:147+5], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
totalQuantity, err := strconv.ParseInt(rawLine[152:152+18], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
totalVolume, err := strconv.ParseInt(rawLine[170:170+18], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
preExe, err := strconv.ParseInt(rawLine[188:188+13], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
indOpc, err := strconv.ParseInt(rawLine[201:201+1], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
expirationDate, err := time.Parse("20060102 -0700", fmt.Sprintf("%s -0300", rawLine[202:202+8]))
if err != nil {
return nil, err
return nil, nil, err
}
fatCot, err := strconv.ParseInt(rawLine[210:210+7], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
ptoExe, err := strconv.ParseInt(rawLine[210:210+7], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
distributionNumber, err := strconv.ParseInt(rawLine[242:242+3], 10, 64)
if err != nil {
return nil, err
return nil, nil, err
}
return &AssetInfo{

allData := AssetInfo{
Year: year,
TipReg: int(tipReg),
DataCollectionDate: date,
Expand Down Expand Up @@ -116,21 +117,41 @@ func parseContentLine(rawLine string, year int) (*AssetInfo, error) {
PtoExe: int(ptoExe),
ISINCode: strings.TrimSpace(rawLine[230 : 230+12]),
DistributionNumber: int(distributionNumber),
}, nil
}

return &Asset{
Ticker: allData.Ticker,
CompanyName: allData.CompanyName,
MarketType: allData.MarketType,
SecurityType: allData.SecurityType,
Currency: allData.Currency,
}, &Price{
Year: allData.Year,
DataCollectionDate: allData.DataCollectionDate,
Ticker: allData.Ticker,
FutureMarketExpiration: allData.FutureMarketExpiration,
PriceClose: allData.PriceClose,
TotalVolume: allData.TotalVolume,
PreExe: allData.PreExe,
ExpirationDate: allData.ExpirationDate,
},
nil
}

func parseHistoricData(rawData []string, year int) ([]AssetInfo, error) {
contentList := make([]AssetInfo, len(rawData)-3)
func parseHistoricData(rawData []string, year int) (map[string]*Asset, []Price, error) {
priceList := make([]Price, len(rawData)-3)
assetMap := map[string]*Asset{}
for i, rawLine := range rawData[1 : len(rawData)-2] {
slice, err := parseContentLine(rawLine, year)
assetSlice, priceSlice, err := parseContentLine(rawLine, year)
if err != nil {
return nil, err
return nil, nil, err
}
contentList[i] = *slice
priceList[i] = *priceSlice
assetMap[assetSlice.Ticker] = assetSlice
}
return contentList, nil
return assetMap, priceList, nil
}

func parseHistoricDataFromBytes(data []byte, year int) ([]AssetInfo, error) {
func parseHistoricDataFromBytes(data []byte, year int) (map[string]*Asset, []Price, error) {
return parseHistoricData(strings.Split(string(data), "\n"), year)
}

0 comments on commit caa82fd

Please sign in to comment.