Add cmd to migrate ds from Redis to Pg.

This is work-in-progress, still need to refactor all the tests.

Cargo.lock

@@ -570,7 +570,7 @@ dependencies = [
 
 [[package]]
 name = "backend"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
  "aes-kw",
  "anyhow",
@@ -787,7 +787,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chirpstack"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
  "aes",
  "anyhow",
@@ -874,7 +874,7 @@ dependencies = [
 
 [[package]]
 name = "chirpstack_api"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
  "diesel",
  "hex",
@@ -2434,7 +2434,7 @@ dependencies = [
 
 [[package]]
 name = "lrwn"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
  "aes",
  "anyhow",
@@ -2448,7 +2448,7 @@ dependencies = [
 
 [[package]]
 name = "lrwn_filters"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
  "hex",
  "lrwn",

api/grpc-web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chirpstack/chirpstack-api-grpc-web",
-  "version": "4.7.0-test.4",
+  "version": "4.7.0-test.3",
   "description": "Chirpstack gRPC-web API",
   "license": "MIT",
   "devDependencies": {

api/java/build.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = "io.chirpstack"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 
 repositories {
     mavenCentral()

api/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chirpstack/chirpstack-api",
-  "version": "4.7.0-test.4",
+  "version": "4.7.0-test.3",
   "description": "Chirpstack JS and TS API",
   "license": "MIT",
   "devDependencies": {

api/kotlin/build.gradle.kts

@@ -9,7 +9,7 @@ plugins {
 }
 
 group = "io.chirpstack"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 
 repositories {
     mavenCentral()

api/python/proto/chirpstack-api/internal/internal.proto

@@ -7,9 +7,15 @@ import "chirpstack-api/gw/gw.proto";
 import "google/protobuf/timestamp.proto";
 
 message DeviceSession {
+  // Device EUI.
+  bytes dev_eui = 1;
+
   // Device address.
   bytes dev_addr = 2;
 
+  // Join EUI.
+  bytes join_eui = 3;
+
   // LoRaWAN mac-version.
   common.MacVersion mac_version = 4;
 

api/python/src/setup.py

@@ -18,7 +18,7 @@ CLASSIFIERS = [
 
 setup(
     name='chirpstack-api',
-    version = "4.7.0-test.4",
+    version = "4.7.0-test.3",
     url='https://github.com/brocaar/chirpstack-api',
     author='Orne Brocaar',
     author_email='info@brocaar.com',

api/rust/Cargo.lock

@@ -149,12 +149,6 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
 
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
 [[package]]
 name = "bytes"
 version = "1.4.0"
@@ -178,9 +172,8 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chirpstack_api"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 dependencies = [
- "diesel",
  "hex",
  "pbjson",
  "pbjson-build",
@@ -203,39 +196,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "diesel"
-version = "2.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
-dependencies = [
- "bitflags 2.4.0",
- "byteorder",
- "diesel_derives",
- "itoa",
-]
-
-[[package]]
-name = "diesel_derives"
-version = "2.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44"
-dependencies = [
- "diesel_table_macro_syntax",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "diesel_table_macro_syntax"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
-dependencies = [
- "syn",
-]
-
 [[package]]
 name = "dirs"
 version = "5.0.1"
@@ -497,6 +457,15 @@ dependencies = [
  "hashbrown 0.14.0",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.11.0"
@@ -621,7 +590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735"
 dependencies = [
  "heck",
- "itertools",
+ "itertools 0.11.0",
  "prost",
  "prost-types",
 ]
@@ -732,7 +701,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac"
 dependencies = [
  "bytes",
  "heck",
- "itertools",
+ "itertools 0.10.5",
  "log",
  "multimap",
  "once_cell",
@@ -753,7 +722,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32"
 dependencies = [
  "anyhow",
- "itertools",
+ "itertools 0.10.5",
  "proc-macro2",
  "quote",
  "syn",

api/rust/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "chirpstack_api"
 description = "ChirpStack Protobuf / gRPC API definitions."
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 license = "MIT"
 homepage = "https://www.chirpstack.io"

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "backend"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2018"
 publish = false

chirpstack/Cargo.toml

@@ -3,7 +3,7 @@ name = "chirpstack"
 description = "ChirpStack is an open-source LoRaWAN(TM) Network Server"
 repository = "https://github.com/chirpstack/chirpstack"
 homepage = "https://www.chirpstack.io/"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2021"
 publish = false

chirpstack/src/api/oidc.rs

@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use std::str::FromStr;
 
 use anyhow::{Context, Result};
@@ -8,13 +7,12 @@ use openidconnect::core::{
     CoreResponseType,
 };
 use openidconnect::reqwest::async_http_client;
-use openidconnect::{AdditionalClaims, UserInfoClaims};
 use openidconnect::{
     AuthenticationFlow, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
     OAuth2TokenResponse, RedirectUrl, Scope,
 };
+use openidconnect::{EmptyAdditionalClaims, UserInfoClaims};
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
 use tracing::{error, trace};
 use warp::{Rejection, Reply};
 
@@ -22,15 +20,7 @@ use crate::config;
 use crate::helpers::errors::PrintFullError;
 use crate::storage::{get_async_redis_conn, redis_key};
 
-pub type User = UserInfoClaims<CustomClaims, CoreGenderClaim>;
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CustomClaims {
-    #[serde(flatten)]
-    other: HashMap<String, Value>,
-}
-
-impl AdditionalClaims for CustomClaims {}
+pub type User = UserInfoClaims<EmptyAdditionalClaims, CoreGenderClaim>;
 
 #[derive(Serialize, Deserialize)]
 pub struct CallbackArgs {

chirpstack/src/cmd/configfile.rs

@@ -185,6 +185,12 @@ pub fn run() {
     {{/each}}
   ]
 
+  # Device session expiration.
+  #
+  # The TTL value defines the time after which a device-session expires
+  # after no activity.
+  device_session_ttl="{{ network.device_session_ttl }}"
+
   # Time to wait for uplink de-duplication.
   #
   # This is the time that ChirpStack will wait for other gateways to receive

chirpstack/src/downlink/data.rs

@@ -363,7 +363,12 @@ impl Data {
 
         let gw_down = helpers::select_downlink_gateway(
             Some(self.tenant.id),
-            &self.device.get_device_session()?.region_config_id,
+            &self
+                .device
+                .device_session
+                .as_ref()
+                .unwrap()
+                .region_config_id,
             self.network_conf.gateway_prefer_min_margin,
             self.device_gateway_rx_info.as_mut().unwrap(),
         )?;

chirpstack/src/uplink/data.rs

@@ -585,7 +585,13 @@ impl Data {
 
     fn decrypt_f_opts_mac_commands(&mut self) -> Result<()> {
         trace!("Decrypting mac-commands");
-        let ds = self.device.as_ref().unwrap().get_device_session()?;
+        let ds = self
+            .device
+            .as_ref()
+            .unwrap()
+            .device_session
+            .as_ref()
+            .unwrap();
         if ds.mac_version().to_string().starts_with("1.0") {
             if let Err(e) = self.phy_payload.decode_f_opts_to_mac_commands() {
                 // This avoids failing in case of a corrupted mac-command in the frm_payload.
@@ -604,7 +610,13 @@ impl Data {
 
     fn decrypt_frm_payload(&mut self) -> Result<()> {
         trace!("Decrypting FRMPayload");
-        let ds = self.device.as_ref().unwrap().get_device_session()?;
+        let ds = self
+            .device
+            .as_ref()
+            .unwrap()
+            .device_session
+            .as_ref()
+            .unwrap();
         let mut f_port = 0;
 
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
@@ -644,7 +656,13 @@ impl Data {
 
     fn set_adr(&mut self) -> Result<()> {
         trace!("Set ADR flag in device-session");
-        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
+        let ds = self
+            .device
+            .as_mut()
+            .unwrap()
+            .device_session
+            .as_mut()
+            .unwrap();
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
             ds.adr = pl.fhdr.f_ctrl.adr;
         }
@@ -772,7 +790,7 @@ impl Data {
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
             if pl.fhdr.f_ctrl.adr_ack_req {
                 let region_conf = region::get(&self.uplink_frame_set.region_config_id)?;
-                let ds = d.get_device_session_mut()?;
+                let ds = d.device_session.as_mut().unwrap();
 
                 // We reset the device-session enabled_uplink_channel_indices and
                 // extra_uplink_channels. On the downlink path, the mac-command handling will
@@ -843,7 +861,13 @@ impl Data {
     }
 
     fn append_meta_data_to_uplink_history(&mut self) -> Result<()> {
-        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
+        let ds = self
+            .device
+            .as_mut()
+            .unwrap()
+            .device_session
+            .as_mut()
+            .unwrap();
 
         // ignore re-transmissions we don't know the source of the
         // re-transmission (it might be a replay-attack)
@@ -888,7 +912,13 @@ impl Data {
     fn append_meta_data_to_uplink_history_relayed(&mut self) -> Result<()> {
         trace!("Apping meta-data of relayed uplink to upink history");
 
-        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
+        let ds = self
+            .device
+            .as_mut()
+            .unwrap()
+            .device_session
+            .as_mut()
+            .unwrap();
         let relay_ctx = self.relay_context.as_ref().unwrap();
 
         // ignore re-transmissions we don't know the source of the
@@ -1365,10 +1395,13 @@ impl Data {
     }
 
     fn _is_end_to_end_encrypted(&self) -> bool {
-        let ds = match self.device.as_ref().unwrap().get_device_session() {
-            Ok(v) => v,
-            Err(_) => return false,
-        };
+        let ds = self
+            .device
+            .as_ref()
+            .unwrap()
+            .device_session
+            .as_ref()
+            .unwrap();
 
         if !ds.js_session_key_id.is_empty() {
             return true;

chirpstack/src/uplink/join_sns.rs

@@ -641,7 +641,13 @@ impl JoinRequest {
     async fn update_device(&mut self) -> Result<()> {
         trace!("Updating device");
         let dp = self.device_profile.as_ref().unwrap();
-        let ds = self.device.as_ref().unwrap().get_device_session()?;
+        let ds = self
+            .device
+            .as_ref()
+            .unwrap()
+            .device_session
+            .as_ref()
+            .unwrap();
 
         self.device = Some(
             device::partial_update(

lrwn-filters/Cargo.toml

@@ -3,7 +3,7 @@ name = "lrwn_filters"
 description = "Library for filtering LoRaWAN payloads on DevAddr and JoinEUIs prefixes"
 homepage = "https://www.chirpstack.io/"
 license = "MIT"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2021"
 repository = "https://github.com/chirpstack/chirpstack"

lrwn/Cargo.toml

@@ -3,7 +3,7 @@ name = "lrwn"
 description = "Library for encoding / decoding LoRaWAN frames."
 homepage = "https://www.chirpstack.io"
 license = "MIT"
-version = "4.7.0-test.4"
+version = "4.7.0-test.3"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2018"
 repository = "https://github.com/chirpstack/chirpstack"

ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chirpstack-ui",
-  "version": "4.7.0-test.4",
+  "version": "4.7.0-test.3",
   "private": true,
   "dependencies": {
     "@ant-design/colors": "^7.0.0",