0.0.1.6
This commit is contained in:
1 parent
97649d73eb
commit
e77afd4d29
127 files changed
+7320
-1886
No files matched your search
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.setString(3, rnd);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"\nformat=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.addError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String lettreSuivante(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.setString(2, c);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnd) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.newError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q="+ rnd +"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.incrementError(name); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), Rnds.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
-1
@@ -1,2 +1,2 @@
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1785979565653" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1785979565653" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre"/>\n </vmType>\n</vmSettings>\n
|
||||
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1791126493219" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1791126493219" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre"/>\n </vmType>\n</vmSettings>\n
|
||||
@@ -0,0 +1,2 @@
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.ui.console.zoomFontHeights=org.eclipse.debug.ui.ProcessConsoleType\=10|0
|
||||
@@ -1,6 +1,6 @@
|
||||
IMPORT_FILES_AND_FOLDERS_RELATIVE=true
|
||||
IMPORT_FILES_AND_FOLDERS_TYPE=23,1
|
||||
eclipse.preferences.version=1
|
||||
platformState=945730117414574
|
||||
platformState=943949841082661
|
||||
quickStart=false
|
||||
tipsAndTricks=true
|
||||
@@ -6,6 +6,7 @@
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
|
||||
<listEntry value="1"/>
|
||||
</listAttribute>
|
||||
<stringAttribute key="org.eclipse.debug.core.terminate.timestamp" value="1791670329354"/>
|
||||
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.sillyjirafe.linker.Main"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MODULE_NAME" value="SillyBlahaj.org"/>
|
||||
|
||||
@@ -10,16 +10,16 @@
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.eclemma.ui.launchGroup.coverage">
|
||||
<mruHistory>
|
||||
<launch memento="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <launchConfiguration local="true" path="Main (3)"/> "/>
|
||||
</mruHistory>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.debug.ui.launchGroup.run">
|
||||
<mruHistory>
|
||||
<launch memento="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <launchConfiguration local="true" path="Main (3)"/> "/>
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
#Eclipse last used NL locale
|
||||
#Sat Oct 10 23:02:27 CEST 2026
|
||||
eclipse.last.nl=en_US
|
||||
File diff suppressed because it is too large.
Load diff
Binary file not shown.
Binary file not shown.
File renamed without changes.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
File renamed without changes.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,7 +1,9 @@
|
||||
INDEX VERSION 1.134+/home/kotama/Silly-Jirafe/.metadata/.plugins/org.eclipse.jdt.core
|
||||
1568060524.index
|
||||
4203645977.index
|
||||
1865797976.index
|
||||
206920101.index
|
||||
INDEX VERSION 1.134+/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.jdt.core
|
||||
263131634.index
|
||||
32071656.index
|
||||
2407587206.index
|
||||
2916335905.index
|
||||
3079814413.index
|
||||
3263143359.index
|
||||
206920101.index
|
||||
2910310629.index
|
||||
1865797976.index
|
||||
Binary file not shown.
@@ -2,4 +2,6 @@
|
||||
<dirs>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" stamp="0"/>
|
||||
</dirs>
|
||||
@@ -12,4 +12,16 @@
|
||||
<entry path="null"/>
|
||||
</endorsedDirs>
|
||||
</libraryInfo>
|
||||
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" version="25.0.4.1">
|
||||
<bootpath>
|
||||
<entry path="null"/>
|
||||
</bootpath>
|
||||
<extensionDirs>
|
||||
<entry path="null"/>
|
||||
</extensionDirs>
|
||||
<endorsedDirs>
|
||||
<entry path="null"/>
|
||||
</endorsedDirs>
|
||||
</libraryInfo>
|
||||
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" version="25.0.4.1"/>
|
||||
</libraryInfos>
|
||||
@@ -2,13 +2,13 @@
|
||||
<section name="Workbench">
|
||||
<item key="org.eclipse.jdt.ui.last.selected.jre.kind2" value="2"/>
|
||||
<item key="org.eclipse.jdt.ui.last.selected.execution.enviroment" value="JavaSE-21"/>
|
||||
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/kotama/Silly-Jirafe/libs"/>
|
||||
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/libs"/>
|
||||
<section name="org.eclipse.jdt.internal.ui.packageview.PackageExplorerPart">
|
||||
<item key="group_libraries" value="true"/>
|
||||
<item key="layout" value="2"/>
|
||||
<item key="rootMode" value="1"/>
|
||||
<item key="linkWithEditor" value="false"/>
|
||||
<item key="memento" value="<?xml version="1.0" encoding="UTF-8"?>
<packageExplorer group_libraries="1" layout="2" linkWithEditor="0" rootMode="1" workingSetName="Aggregate for window 1785979556746">
<customFilters userDefinedPatternsEnabled="false">
<xmlDefinedFilters>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.StaticsFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder" isEnabled="true"/>
<child filterId="org.eclipse.mylyn.java.ui.MembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter" isEnabled="true"/>
<child filterId="org.eclipse.m2e.MavenModuleFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.FieldsFilter" isEnabled="false"/>
</xmlDefinedFilters>
</customFilters>
</packageExplorer>"/>
|
||||
<item key="memento" value="<?xml version="1.0" encoding="UTF-8"?>
<packageExplorer group_libraries="1" layout="2" linkWithEditor="0" rootMode="1" workingSetName="Aggregate for window 1785979556746">
<customFilters userDefinedPatternsEnabled="false">
<xmlDefinedFilters>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.StaticsFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder" isEnabled="true"/>
<child filterId="org.eclipse.mylyn.java.ui.MembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.m2e.MavenModuleFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter" isEnabled="true"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.FieldsFilter" isEnabled="false"/>
</xmlDefinedFilters>
</customFilters>
</packageExplorer>"/>
|
||||
</section>
|
||||
<section name="JavaProjectWizard.dialogBounds">
|
||||
<item key="DIALOG_X_ORIGIN" value="0"/>
|
||||
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<session version="1.0">
|
||||
<refactoring comment="Copy element 'libs' to 'sillyjirafe.org.api'
- Original project: 'SillyJirafe.org'
- Destination element: 'sillyjirafe.org.api'
- Original element: 'libs'" description="Copy folder" destination="=sillyjirafe.org.api" element1="libs" files="0" flags="589830" folders="1" id="org.eclipse.jdt.ui.copy" policy="org.eclipse.jdt.ui.copyResources" stamp="1791564728729" units="0" version="1.0"/>
|
||||
</session>
|
||||
+1
@@ -0,0 +1 @@
|
||||
1791564728729 Copy folder
|
||||
@@ -39,3 +39,11 @@
|
||||
2026-09-22 16:37:19,751 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
2026-09-25 09:23:41,947 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-09-29 23:22:34,391 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-04 17:08:14,297 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-04 22:27:35,351 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-06 13:16:34,293 [Worker-5: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
2026-10-07 00:08:16,911 [Worker-8: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-07 16:37:42,461 [Worker-2: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-07 21:50:08,486 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-09 18:50:09,616 [Worker-9: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-10 23:02:35,211 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
@@ -0,0 +1,41 @@
|
||||
<configuration scan="true">
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date [%thread] %-5level %logger{35} - %msg%n</pattern>
|
||||
</encoder>
|
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
|
||||
<level>${org.eclipse.m2e.log.console.threshold:-OFF}</level> <!-- change to DEBUG to mimic '-consolelog' behaviour -->
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<File>${org.eclipse.m2e.log.dir}/0.log</File>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<FileNamePattern>${org.eclipse.m2e.log.dir}/%i.log</FileNamePattern>
|
||||
<MinIndex>1</MinIndex>
|
||||
<MaxIndex>10</MaxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<MaxFileSize>10MB</MaxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date [%thread] %-5level %logger{35} - %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="EclipseLog" class="org.eclipse.m2e.logback.appender.EclipseLogAppender">
|
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
|
||||
<level>WARN</level>
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<appender name="MavenConsoleLog" class="org.eclipse.m2e.logback.appender.MavenConsoleAppender">
|
||||
</appender>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="FILE" />
|
||||
<appender-ref ref="STDOUT" />
|
||||
<appender-ref ref="EclipseLog" />
|
||||
<appender-ref ref="MavenConsoleLog" />
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -3,4 +3,5 @@
|
||||
<workingSet editPageId="org.eclipse.jdt.internal.ui.DynamicSourcesWorkingSet" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979555150_0" label="Java Main Sources" name="Java Main Sources"/>
|
||||
<workingSet editPageId="org.eclipse.jdt.internal.ui.DynamicSourcesWorkingSet" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979555168_1" label="Java Test Sources" name="Java Test Sources"/>
|
||||
<workingSet aggregate="true" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979556746_2" label="Window Working Set" name="Aggregate for window 1785979556746"/>
|
||||
<workingSet aggregate="true" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1791324496308_3" label="Window Working Set" name="Aggregate for window 1791324496308"/>
|
||||
</workingSetManager>
|
||||
Reference in new issue
Block a user