GithubRepoApi.java

  1. package us.codecraft.webmagic.example;

  2. import us.codecraft.webmagic.Site;
  3. import us.codecraft.webmagic.model.ConsolePageModelPipeline;
  4. import us.codecraft.webmagic.model.HasKey;
  5. import us.codecraft.webmagic.model.OOSpider;
  6. import us.codecraft.webmagic.model.annotation.ExtractBy;
  7. import us.codecraft.webmagic.model.annotation.ExtractByUrl;

  8. import java.util.List;

  9. /**
  10.  * @author code4crafter@gmail.com <br>
  11.  * @since 0.4.1
  12.  */
  13. public class GithubRepoApi implements HasKey {

  14.     @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name", source = ExtractBy.Source.RawText)
  15.     private String name;

  16.     @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..owner.login", source = ExtractBy.Source.RawText)
  17.     private String author;

  18.     @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.language",multi = true, source = ExtractBy.Source.RawText)
  19.     private List<String> language;

  20.     @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.stargazers_count", source = ExtractBy.Source.RawText)
  21.     private int star;

  22.     @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.forks_count", source = ExtractBy.Source.RawText)
  23.     private int fork;

  24.     @ExtractByUrl
  25.     private String url;

  26.     public static void main(String[] args) {
  27.         OOSpider.create(Site.me().setSleepTime(100)
  28.                 , new ConsolePageModelPipeline(), GithubRepoApi.class)
  29.                 .addUrl("https://api.github.com/repos/code4craft/webmagic").run();
  30.     }

  31.     @Override
  32.     public String key() {
  33.         return author + ":" + name;
  34.     }

  35.     public String getName() {
  36.         return name;
  37.     }

  38.     public String getAuthor() {
  39.         return author;
  40.     }

  41.     public List<String> getLanguage() {
  42.         return language;
  43.     }

  44.     public String getUrl() {
  45.         return url;
  46.     }

  47.     public int getStar() {
  48.         return star;
  49.     }

  50.     public int getFork() {
  51.         return fork;
  52.     }
  53. }