This commit is contained in:
Celeste Madeline Samuel Nokto Dur Grossiord committed 2026-10-11 15:47:17 +02:00
1 parent 97649d73eb
commit e77afd4d29
127 files changed
+7320 -1886

No files matched your search

@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,176 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class PixabayImagesApi implements Api
{
public String getName()
{
return "PixabayImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("hits");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("name");
String url = repo.getString("pageURL");
String licence = "cc0";
resultats.add(new element(titre, url, licence, "pixabay", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,151 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.setString(3, rnd);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"\nformat=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,41 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,164 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String actuelle) {
char[] lettres = actuelle.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,138 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.addError(name);
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,49 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read());
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,98 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,175 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,175 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,164 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String lettreSuivante(String actuelle) {
char[] lettres = actuelle.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,99 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,177 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.setString(2, c);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,50 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
Map<String, Boolean> Succes = new HashMap<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (Api api : Apis) {
try {
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()));
Elements.addAll(currentapi);
Succes.put(api.getName(), true);
} catch (Exception e) {
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
Succes.put(api.getName(), false);
}
}
return Elements;
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,50 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
Map<String, Boolean> Succes = new HashMap<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (Api api : Apis) {
try {
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
Elements.addAll(currentapi);
Succes.put(api.getName(), true);
} catch (Exception e) {
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
Succes.put(api.getName(), false);
}
}
return Elements;
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read());
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,177 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
import java.util.ArrayList;
import java.util.List;
public class GitHubMITApi implements Api
{
public String getName()
{
return "GitHubMITApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,139 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,49 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (Succes.getOrDefault(nom, false)) {
crawlstateDB.updatePage(nom, ...); // succès normal, avance
} else {
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnd) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,54 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (Succes.getOrDefault(nom, false)) {
crawlstateDB.updatePage(nom, ...); // succès normal, avance
} else {
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,98 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
import java.util.ArrayList;
import java.util.List;
public class GitHubMITApi implements Api
{
public String getName()
{
return "GitHubMITApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,110 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,99 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.newError(name);
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/audio/?q="+ rnd +"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,139 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class PixabayImagesApi implements Api
{
public String getName()
{
return "PixabayImagesApi";
}
public String getURL(int page, String apikey) {
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("hits");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("name");
String url = repo.getString("pageURL");
String licence = "cc0";
resultats.add(new element(titre, url, licence, "pixabay", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.incrementError(name); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 1;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), Rnds.get(Apis.get(i).getName()));
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}