0.0.1.6
This commit is contained in:
1 parent
97649d73eb
commit
e77afd4d29
127 files changed
+7320
-1886
No files matched your search
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.setString(3, rnd);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"\nformat=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.addError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String lettreSuivante(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.setString(2, c);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnd) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.newError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q="+ rnd +"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.incrementError(name); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), Rnds.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user