Golang scraper to get data from Google Play Store

google-play-scraper

GoDoc Go Report Card Coverage Status

Golang scraper to get data from Google Play Store

This project is inspired by the google-play-scraper node.js project

Installation

go get -u github.com/n0madic/google-play-scraper/...

Usage

Get app details

Retrieves the full detail of an application.

package main

import (
    "github.com/davecgh/go-spew/spew"
    "github.com/n0madic/google-play-scraper/pkg/app"
)

func main() {
    a := app.New("com.google.android.googlequicksearchbox", app.Options{
        Country:  "us",
        Language: "us",
    })
    err := a.LoadDetails()
    if err != nil {
        panic(err)
    }
    err = a.LoadPermissions()
    if err != nil {
        panic(err)
    }
    spew.Dump(a)
}

Search apps

Retrieves a list of apps that results of searching by the given term.

0 { panic(errors[0]) } for _, app := range query.Results { if !app.IAPOffers { fmt.Println(app.Title, app.URL) } } }">
package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/search"
)

func main() {
    query := search.NewQuery("game", search.PricePaid,
        search.Options{
            Country:  "ru",
            Language: "us",
            Number:   100,
            Discount: true,
            PriceMax: 100,
            ScoreMin: 4,
        })

    err := query.Run()
    if err != nil {
        panic(err)
    }

    errors := query.LoadMoreDetails(20)
    if len(errors) > 0 {
        panic(errors[0])
    }

    for _, app := range query.Results {
        if !app.IAPOffers {
            fmt.Println(app.Title, app.URL)
        }
    }
}

Get category

Returns a list of clusters for the specified application category.

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/category"
    "github.com/n0madic/google-play-scraper/pkg/store"
)

func main() {
    clusters, err := category.New(store.Game, store.SortNewest, store.AgeFiveUnder, category.Options{
        Language: "us",
        Number:   100,
    })
    if err != nil {
        panic(err)
    }

    err = clusters["Top New Free Games"].Run()
    if err != nil {
        panic(err)
    }

    for _, app := range clusters["Top New Free Games"].Results {
        fmt.Println(app.Title, app.URL)
    }
}

Get collection

Retrieve a list of applications from one of the collections at Google Play.

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/collection"
    "github.com/n0madic/google-play-scraper/pkg/store"
)

func main() {
    c := collection.New(store.TopNewPaid, collection.Options{
        Country: "uk",
        Number:  100,
    })
    err := c.Run()
    if err != nil {
        panic(err)
    }

    for _, app := range c.Results {
        fmt.Println(app.Title, app.Price, app.URL)
    }
}

Get developer applications

Returns the list of applications by the given developer name or ID

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/developer"
)

func main() {
    dev := developer.New("Google LLC", developer.Options{
        Number: 100,
    })
    err := dev.Run()
    if err != nil {
        panic(err)
    }

    for _, app := range dev.Results {
        fmt.Println(app.Title, "by", app.Developer, app.URL)
    }
}

Get reviews

Retrieves a page of reviews for a specific application.

Note that this method returns reviews in a specific language (english by default), so you need to try different languages to get more reviews. Also, the counter displayed in the Google Play page refers to the total number of 1-5 stars ratings the application has, not the written reviews count. So if the app has 100k ratings, don't expect to get 100k reviews by using this method.

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/reviews"
)

func main() {
    r := reviews.New("com.activision.callofduty.shooter", reviews.Options{
        Number: 100,
    })

    err := r.Run()
    if err != nil {
        panic(err)
    }

    for _, review := range r.Results {
        fmt.Println(review.Score, review.Text)
    }
}

Get similar

Returns a list of similar apps to the one specified.

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/similar"
)

func main() {
    sim := similar.New("com.android.chrome", similar.Options{
        Number: 100,
    })
    err := sim.Run()
    if err != nil {
        panic(err)
    }

    for _, app := range sim.Results {
        fmt.Println(app.Title, app.URL)
    }
}

Get suggest

Given a string returns up to five suggestion to complete a search query term.

package main

import (
    "fmt"

    "github.com/n0madic/google-play-scraper/pkg/suggest"
)

func main() {
    sug, err := suggest.Get("chrome", suggest.Options{
        Country:  "us",
        Language: "us",
    })
    if err != nil {
        panic(err)
    }

    for _, s := range sug {
        fmt.Println(s)
    }
}
Comments
  • Bringing incorrect data after google play ui update

    Bringing incorrect data after google play ui update

    I would like to know if it will be remapped to a new interface or if there is a solution to this problem

    { "AdSupported": false, "AndroidVersion": "[[null,[[1800],null,[72000]]]]", "AndroidVersionMin": 0, "ContentRating": "", "ContentRatingDescription": "", "Description": "", "DescriptionHTML": "", "Developer": "", "DeveloperAddress": "", "DeveloperEmail": "", "DeveloperID": "", "DeveloperURL": "https://play.google.com", "DeveloperWebsite": "", "FamilyGenre": "", "FamilyGenreID": "", "Free": true, "Genre": "", "GenreID": "", "HeaderImage": "", "IAPOffers": false, "IAPRange": "", "Icon": "", "ID": "com.whatsapp",

  • fix: 🐛 fixed mapping for developer information

    fix: 🐛 fixed mapping for developer information

    👉 With the new update of the play store frontend, mapping has been changed very much. 👉 developerById and developerByName has different different mappings now. 👉 The batchExecute method still works with old mappings.

  • App No Infomation

    App No Infomation

    app := New("com.kgi", Options{"tw", "zh-TW"}) err := app.LoadDetails() if err != nil { t.Error(err) }

    can't get all detail information

  • Review collection stucks when there's no enough reviews

    Review collection stucks when there's no enough reviews

    I was testing the code about collecting of new reviews with following configuration:

    r := reviews.New("com.kolayrandevu.isletme", reviews.Options{ Number: 100, Sorting: store.SortNewest, Language: "zh", })

    When I run this, the code first collects the initial reviews successfully for 2 existing reviews on Google Play. Later on, the following code in reviews.go creates an infinite loop because the number property I've provided is 100 but the available reviews on Google Play is only 2.

    for len(reviews.Results) != reviews.options.Number {

  • Overall rating numbers and Avg Rating

    Overall rating numbers and Avg Rating

    Hi, Great library and thank you. I was looking for a Google Scraper and found one.

    Is there any way to extract the overall number and the average as well? In other words, (from the screenshot below), it'd be nice to extract the 4.3 overall ratings and the distribution numbers, for example for rating 3 the following screenshot presents on hover with 1,137 reviews.

    I did try and get all the reviews below

    	r := reviews.New("xxx.application.ignore.it", reviews.Options{
    		Number:   1000000,
    		Language: "ja",
    	})
    
    	err := r.Run()
    	if err != nil {
    		panic(err)
    	}
    
    	for _, review := range r.Results {
    		// fmt.Println(review.Reviewer, review.Timestamp, review.Score, review.Text)
    		fmt.Println(review.Score)
    	}
    
    ╰─$ go run main.go |grep 3|wc -l
         633
    

    This one doesn't give exact number which is ok since it doesn't extract 100% of the reviews.

    Screen Shot 2022-06-10 at 21 06 28

  • Bump github.com/tidwall/gjson from 1.8.1 to 1.9.3

    Bump github.com/tidwall/gjson from 1.8.1 to 1.9.3

    Bumps github.com/tidwall/gjson from 1.8.1 to 1.9.3.

    Commits

    Dependabot compatibility score

    Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting @dependabot rebase.


    Dependabot commands and options

    You can trigger Dependabot actions by commenting on this PR:

    • @dependabot rebase will rebase this PR
    • @dependabot recreate will recreate this PR, overwriting any edits that have been made to it
    • @dependabot merge will merge this PR after your CI passes on it
    • @dependabot squash and merge will squash and merge this PR after your CI passes on it
    • @dependabot cancel merge will cancel a previously requested merge and block automerging
    • @dependabot reopen will reopen this PR if it is closed
    • @dependabot close will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
    • @dependabot ignore this major version will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
    • @dependabot ignore this minor version will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
    • @dependabot ignore this dependency will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
    • @dependabot use these labels will set the current labels as the default for future PRs for this repo and language
    • @dependabot use these reviewers will set the current reviewers as the default for future PRs for this repo and language
    • @dependabot use these assignees will set the current assignees as the default for future PRs for this repo and language
    • @dependabot use this milestone will set the current milestone as the default for future PRs for this repo and language

    You can disable automated security fix PRs for this repo from the Security Alerts page.

  • Bump github.com/tidwall/gjson from 1.6.1 to 1.6.5

    Bump github.com/tidwall/gjson from 1.6.1 to 1.6.5

    Bumps github.com/tidwall/gjson from 1.6.1 to 1.6.5.

    Commits

    Dependabot compatibility score

    Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting @dependabot rebase.


    Dependabot commands and options

    You can trigger Dependabot actions by commenting on this PR:

    • @dependabot rebase will rebase this PR
    • @dependabot recreate will recreate this PR, overwriting any edits that have been made to it
    • @dependabot merge will merge this PR after your CI passes on it
    • @dependabot squash and merge will squash and merge this PR after your CI passes on it
    • @dependabot cancel merge will cancel a previously requested merge and block automerging
    • @dependabot reopen will reopen this PR if it is closed
    • @dependabot close will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
    • @dependabot ignore this major version will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
    • @dependabot ignore this minor version will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
    • @dependabot ignore this dependency will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
    • @dependabot use these labels will set the current labels as the default for future PRs for this repo and language
    • @dependabot use these reviewers will set the current reviewers as the default for future PRs for this repo and language
    • @dependabot use these assignees will set the current assignees as the default for future PRs for this repo and language
    • @dependabot use this milestone will set the current milestone as the default for future PRs for this repo and language

    You can disable automated security fix PRs for this repo from the Security Alerts page.

Golang based web site opengraph data scraper with caching
Golang based web site opengraph data scraper with caching

Snapper A Web microservice for capturing a website's OpenGraph data built in Golang Building Snapper building the binary git clone https://github.com/

Oct 5, 2022
Scraper to download school attendance data from the DfE's statistics website
Scraper to download school attendance data from the DfE's statistics website

?? Simple to use. Scrape attendance data with a single command! ?? Super fast. A

Mar 31, 2022
A simple scraper to export data from buildkite to honeycomb using opentelemetry SDK
A simple scraper to export data from buildkite to honeycomb using opentelemetry SDK

A quick scraper program that let you export builds on BuildKite as OpenTelemetry data and then send them to honeycomb.io for slice-n-dice high cardinality analysis.

Jul 7, 2022
Elegant Scraper and Crawler Framework for Golang

Colly Lightning Fast and Elegant Scraping Framework for Gophers Colly provides a clean interface to write any kind of crawler/scraper/spider. With Col

Jan 9, 2023
A crawler/scraper based on golang + colly, configurable via JSON

A crawler/scraper based on golang + colly, configurable via JSON

Aug 21, 2022
A crawler/scraper based on golang + colly, configurable via JSON

Super-Simple Scraper This a very thin layer on top of Colly which allows configuration from a JSON file. The output is JSONL which is ready to be impo

Aug 21, 2022
Warhammer40K faction scraper written in Golang, powered by colly.

Wascra Description Wascra is a tool written in Golang, which lets you extract all relevant Datasheet info from a Warhammer40K (9th edition) faction fr

Feb 8, 2022
Web Scraper in Go, similar to BeautifulSoup

soup Web Scraper in Go, similar to BeautifulSoup soup is a small web scraper package for Go, with its interface highly similar to that of BeautifulSou

Jan 9, 2023
Simple price scraper with HTTP server/exporter for use with Prometheus

priceserver v0.3 Simple price scraper with HTTP server/exporter for use with Prometheus Currently working with Bitrue.com exchange but easily adaptabl

Nov 16, 2021
A cli scraper of gocomics.com made in go

goComic goComic is a cli tool written in go that scrapes your favorite childhood favorite comic from gocomics.com. It will give you a single days comi

Dec 24, 2021
Best Room Price Scraper from Booking.com

Best Room Price Scraper from Booking.com This repo is a tutorial of Large Scale

Nov 11, 2022
Go-based search engine URL collector , support Google, Bing, can be based on Google syntax batch collection URL
Go-based search engine URL collector , support Google, Bing, can be based on Google syntax batch collection URL

Go-based search engine URL collector , support Google, Bing, can be based on Google syntax batch collection URL

Nov 9, 2022
DorkScout - Golang tool to automate google dork scan against the entiere internet or specific targets
DorkScout - Golang tool to automate google dork scan against the entiere internet or specific targets

dorkscout dokrscout is a tool to automate the finding of vulnerable applications or secret files around the internet throught google searches, dorksco

Nov 21, 2022
🦙 acao(阿草), the tool man for data scraping of https://asoul.video/.

?? acao acao(阿草), the tool man for data scraping of https://asoul.video/. Deploy to Aliyun serverless function with Raika update_member Update A-SOUL

Jul 25, 2022
Go-Yahoo-Finance-Daily-Actives - Scrape for the daily actives on yh Finance and save the data to a CSV, and optionally send it to yourself as an email
Go-Yahoo-Finance-Daily-Actives - Scrape for the daily actives on yh Finance and save the data to a CSV, and optionally send it to yourself as an email

Go-Yahoo-Finance-Daily-Actives - Scrape for the daily actives on yh Finance and save the data to a CSV, and optionally send it to yourself as an email

Dec 13, 2022
View reddit memes and posts from ur terminal with golang and webscraping

goddit View reddit memes and posts from your terminal with golang and webscraping Installation run the following commands on your terminal to install

Feb 22, 2021
Pholcus is a distributed high-concurrency crawler software written in pure golang
Pholcus is a distributed high-concurrency crawler software written in pure golang

Pholcus Pholcus(幽灵蛛)是一款纯 Go 语言编写的支持分布式的高并发爬虫软件,仅用于编程学习与研究。 它支持单机、服务端、客户端三种运行模式,拥有Web、GUI、命令行三种操作界面;规则简单灵活、批量任务并发、输出方式丰富(mysql/mongodb/kafka/csv/excel等

Dec 30, 2022
[爬虫框架 (golang)] An awesome Go concurrent Crawler(spider) framework. The crawler is flexible and modular. It can be expanded to an Individualized crawler easily or you can use the default crawl components only.

go_spider A crawler of vertical communities achieved by GOLANG. Latest stable Release: Version 1.2 (Sep 23, 2014). QQ群号:337344607 Features Concurrent

Jan 6, 2023
New World Auction House Crawler In Golang

New-World-Auction-House-Crawler Goal of this library is to have a process which grabs New World auction house data in the background while playing the

Sep 7, 2022